metadata and regional bucket

Author: lithium147

dataflow-deploy.sh

@@ -1,7 +1,19 @@
-mvn compile exec:java -Dexec.mainClass=com.solubris.Scrape \
--Dexec.args="--runner=DataflowRunner --project=explore-447815 \
+#!/usr/bin/env bash
+
+source env.sh
+
+gcloud storage buckets create --project="$GCP_PROJECT" --location=$GCP_REGION "$GCP_BUCKET"
+gcloud storage cp metadata.json "$GCP_BUCKET/scrape-1.0.0_metadata"
+
+mvn clean compile exec:java -Dexec.mainClass=com.solubris.Scrape \
+-Dexec.args="--runner=DataflowRunner --project=$GCP_PROJECT \
 --url=https://www.rhs.org.uk/shows-events/rhs-chelsea-flower-show/ticket-options \
---gcpTempLocation=gs://explore-temp-bucket/tmp \
---templateLocation=gs://explore-temp-bucket/scrape-1.0.0 \
---region=europe-west1" \
+--gcpTempLocation=$GCP_BUCKET/tmp \
+--region=$GCP_REGION \
+--templateLocation=$GCP_BUCKET/scrape-1.0.0" \
 -Pdataflow-runner
+
+#gcloud storage cp metadata.json gs://explore-temp-bucket/scrape-1.0.0_metadata
+#gcloud auth login
+#gcloud config set project PROJECT_ID
+

dataflow.sh

@@ -1,6 +1,10 @@
-mvn compile exec:java -Dexec.mainClass=com.solubris.Scrape \
--Dexec.args="--runner=DataflowRunner --project=explore-447815 \
+#!/usr/bin/env bash
+
+source env.sh
+
+mvn clean compile exec:java -Dexec.mainClass=com.solubris.Scrape \
+-Dexec.args="--runner=DataflowRunner --project=$GCP_PROJECT \
 --url=https://www.rhs.org.uk/shows-events/rhs-chelsea-flower-show/ticket-options \
---gcpTempLocation=gs://explore-temp-bucket/tmp \
---region=europe-west1" \
+--gcpTempLocation=$GCP_BUCKET/tmp \
+--region=$GCP_REGION" \
 -Pdataflow-runner

direct.sh

@@ -1,3 +1,5 @@
+#!/usr/bin/env bash
+
 mvn compile exec:java -Dexec.mainClass=com.solubris.Scrape \
 -Dexec.args="--runner=DirectRunner \
 --url=https://www.rhs.org.uk/shows-events/rhs-chelsea-flower-show/ticket-options" \

env.sh

@@ -0,0 +1,3 @@
+export GCP_PROJECT=explore-447815
+export GCP_REGION=europe-west2
+export GCP_BUCKET="gs://explore-$GCP_REGION"

metadata.json

@@ -0,0 +1,12 @@
+{
+  "description": "Scrapes the given URL",
+  "name": "scrape-rhs-new",
+  "streaming": false,
+  "parameters": [
+    {
+      "name": "url",
+      "helpText": "url to be scraped",
+      "label": "url to be scraped"
+    }
+  ]
+}