Added auth script to help switch app default credentials and using

service accounts with these scripts.

Added in a script to rebind output values from a given submission
in case that failed to work properly.

auth.sh

+if [[ $1 =~ \.json$ ]]; then
+    echo "Authorizing service account using json file"
+    gcloud auth activate-service-account --key-file=$1
+    export GOOGLE_APPLICATION_CREDENTIALS=$1
+elif [[ $1 =~ "@" ]]; then
+    echo "Authorizing using email address"
+    gcloud auth login $1
+    export GOOGLE_APPLICATION_CREDENTIALS=~/.config/gcloud/legacy_credentials/$1/adc.json
+else
+    echo "Run this script in the following manner:"
+    echo "   To auth as a regular user:    '. auth.sh <your email>'"
+    echo "   To auth as a service account: '. auth.sh <path to service account json>'"
+    echo "NOTE:  The dot before auth.sh important as this sets an environment variable"
+fi
\ No newline at end of file

bigquery_billing_report/bigquery_billing_report.py

@@ -2,7 +2,7 @@
 from common import *
 
 
-def get_pricing(ws_namespace, ws_name, query_sub_id = None, query_workflow_id = None, show_all_calls=False):
+def get_pricing(ws_namespace, ws_name, query_sub_id, query_workflow_id, show_all_calls, dataset_project_name, dataset_name):
     print "Retrieving submissions in workspace..."       
     workspace_request = firecloud_api.get_workspace(ws_namespace, ws_name)
     
@@ -29,7 +29,7 @@ def get_pricing(ws_namespace, ws_name, query_sub_id = None, query_workflow_id =
 
             workflow_dict[wf_id] = {"submission_id":sub_id, "workflow":wf}
 
-    get_workflow_pricing(ws_namespace, ws_name, workflow_dict, submission_dict, query_workflow_id!=None and len(query_workflow_id) > 0, show_all_calls)
+    get_workflow_pricing(ws_namespace, ws_name, workflow_dict, submission_dict, query_workflow_id!=None and len(query_workflow_id) > 0, show_all_calls, dataset_project_name, dataset_name)
 
 
 class CostRow():
@@ -78,7 +78,7 @@ def _fiss_access_headers_local(headers=None):
     return fiss_headers
 
 
-def get_workflow_pricing(ws_namespace, ws_name, workflow_dict, submission_dict, singleWorkflowMode, show_all_calls):
+def get_workflow_pricing(ws_namespace, ws_name, workflow_dict, submission_dict, singleWorkflowMode, show_all_calls, dataset_project_name, dataset_name):
     firecloud_api._fiss_access_headers = _fiss_access_headers_local
     if len(workflow_dict) == 0:
         fail("No submissions or workflows matching the criteria were found.")
@@ -91,8 +91,8 @@ def get_workflow_pricing(ws_namespace, ws_name, workflow_dict, submission_dict,
     
     print "Gathering pricing data..."
 
-    client = bigquery.Client(ws_namespace)
-    dataset = client.dataset('billing_export')
+    client = bigquery.Client(dataset_project_name)
+    dataset = client.dataset(dataset_name)
 
     matched_workflow_ids = set()
 
@@ -124,7 +124,7 @@ def get_workflow_pricing(ws_namespace, ws_name, workflow_dict, submission_dict,
                              # uncomment for quick testing:
                              #LIMIT 1
                            """ % (dataset.name, table.name, ws_namespace, workflows_subquery)
-
+            print query
             query_results = client.run_sync_query(query)
 
             # Use standard SQL syntax for queries.
@@ -318,14 +318,20 @@ def main():
 
     parser.add_argument('-c', '--calls', dest='show_all_calls', action='store_true', required=False, help='Expand information about each call.')
 
+    parser.add_argument('-dp', '--dataset_project', dest='dataset_project', action='store', required=False, help='Optional project where dataset is stored - defaults to the workspace project.')
+    parser.add_argument('-dn', '--dataset_name', dest='dataset_name', action='store', required=False, help='Optional dataset name where billing data is stored - defaults to billing_export.')
+
     args = parser.parse_args()
 
     if args.workflow_id and not args.submission_id:
         fail("Submission ID must also be provided when querying for a Workflow ID")
 
-    print "Note that this script expects the billing export table to be named 'billing_export'.  "
+    if not args.dataset_name:
+        print "Note that this script expects the billing export table to be named 'billing_export'.  "
 
-    get_pricing(args.ws_namespace, args.ws_name, args.submission_id, args.workflow_id, args.show_all_calls)
+    get_pricing(args.ws_namespace, args.ws_name, args.submission_id, args.workflow_id, args.show_all_calls,
+                args.dataset_project if args.dataset_project else args.ws_namespace,
+                args.dataset_name if args.dataset_name else 'billing_export')
 
 
 if __name__ == "__main__":

common.py

@@ -38,6 +38,8 @@ import dateutil.parser
 
 # firecloud python bindings
 from firecloud import api as firecloud_api
+from firecloud.fccore import __fcconfig as fcconfig
+
 
 processes = []
 
@@ -89,8 +91,7 @@ class DefaultArgsParser:
         args = self.parser.parse_args()
 
         if args.fc_url:
-            firecloud_api.PROD_API_ROOT = args.fc_url
-
+            set_fc_url(args.fc_url)
         return args
 
 
@@ -104,18 +105,42 @@ def list_to_dict(input_list, key_fcn, value_fcn=lambda item: item, transform_fcn
     return dicted_list
 
 
-def setup():
+def get_access_token():
+    return GoogleCredentials.get_application_default().get_access_token().access_token
+
+# only needed until firecloud python library in pypi supports service accounts
+def _fiss_access_headers_local(headers=None):
+    """ Return request headers for fiss.
+        Retrieves an access token with the user's google crededentials, and
+        inserts FISS as the User-Agent.
+    Args:
+        headers (dict): Include additional headers as key-value pairs
+    """
     credentials = GoogleCredentials.get_application_default()
-    print "Using Google client id:", credentials.client_id
+    credentials = credentials.create_scoped(['https://www.googleapis.com/auth/userinfo.profile', 'https://www.googleapis.com/auth/userinfo.email'])
+    access_token = credentials.get_access_token().access_token
+    fiss_headers = {"Authorization" : "bearer " + access_token}
+    fiss_headers["User-Agent"] = firecloud_api.FISS_USER_AGENT
+    if headers:
+        fiss_headers.update(headers)
+    return fiss_headers
+
+
+def setup():
+    firecloud_api._fiss_access_headers = _fiss_access_headers_local
+    registration_info = requests.get("https://api.firecloud.org/register", headers=firecloud_api._fiss_access_headers())
+    if registration_info.status_code == 404:
+        fail("This account is not registered with FireCloud.")
+
+    print "Using credentials for firecloud account:", registration_info.json()["userInfo"]["userEmail"]
 
 
 def get_workflow_metadata(namespace, name, submission_id, workflow_id, *include_keys):
     headers = firecloud_api._fiss_access_headers()
 
     include_key_string = "includeKey=%s&" % ("%2C%20".join(list(include_keys))) if include_keys else ""
-    uri = "{0}/workspaces/{1}/{2}/submissions/{3}/workflows/{4}?&{5}expandSubWorkflows=false".format(
-        firecloud_api.PROD_API_ROOT, namespace, name, submission_id, workflow_id, include_key_string)
-
+    uri = "{0}workspaces/{1}/{2}/submissions/{3}/workflows/{4}?&{5}expandSubWorkflows=false".format(
+        get_fc_url(), namespace, name, submission_id, workflow_id, include_key_string)
 
     return requests.get(uri, headers=headers).json()
 
@@ -142,7 +167,7 @@ def get_entity_by_page(namespace, name, entity_type, page, page_size):
     headers = firecloud_api._fiss_access_headers()
 
     uri = "{0}/workspaces/{1}/{2}/entityQuery/{3}?page={4}&pageSize={5}".format(
-        firecloud_api.PROD_API_ROOT, namespace, name, entity_type, page, page_size)
+        get_fc_url(), namespace, name, entity_type, page, page_size)
 
     return requests.get(uri, headers=headers).json()
 
@@ -171,11 +196,19 @@ def get_all_bound_file_paths(ws_namespace, ws_name):
     return attribute_name_for_url_to_entity_json
 
 
+def set_fc_url(url):
+    fcconfig.root_url = url
+
+
+def get_fc_url():
+    return fcconfig.root_url
+
+
 def get_entity_by_page(namespace, name, entity_type, page, page_size):
     headers = firecloud_api._fiss_access_headers()
 
-    uri = "{0}/workspaces/{1}/{2}/entityQuery/{3}?page={4}&pageSize={5}".format(
-        firecloud_api.PROD_API_ROOT, namespace, name, entity_type, page, page_size)
+    uri = "{0}workspaces/{1}/{2}/entityQuery/{3}?page={4}&pageSize={5}".format(
+        get_fc_url(), namespace, name, entity_type, page, page_size)
 
     return requests.get(uri, headers=headers).json()
 
@@ -184,7 +217,7 @@ def get_workspace_storage_estimate(namespace, name):
     headers = firecloud_api._fiss_access_headers()
 
     uri = "{0}/workspaces/{1}/{2}/storageCostEstimate".format(
-        firecloud_api.PROD_API_ROOT, namespace, name)
+        get_fc_url(), namespace, name)
 
     return requests.get(uri, headers=headers)
 

rebind_output_attributes/README.md

+## Rebind outputs from a given submission
+This script will take a submission id from a given workspace and bind the outputs produced from this submission to the data model using the method config's output expressions.  
+
+Optionally an expression override argument can be given that allows new output expressions to be defined and override the existing method config's output expressions.  This can be used for example to bind an output that did not originally have an expresion defined for it when the analysis was run.
+
+Run this as follows (from the main directory):
+```./run.sh rebind_output_attributes/rebind_output_attributes.py -p <workspace project> -n <workspace name> -s <submission id of the submission you want the outputs from> -e <optional, if used this can override output expressions used in the method config for this submission.  Syntax is in the form {"output_name": "expression"}```
\ No newline at end of file

rebind_output_attributes/rebind_output_attributes.py

+#!/usr/bin/env python
+from common import *
+
+
+def fix_outputs(ws_namespace, ws_name, submission_id, expressions_override):
+    # get the workspace
+    workspace_request = firecloud_api.get_workspace(ws_namespace, ws_name)
+
+    if workspace_request.status_code != 200:
+        fail("Unable to find workspace: %s/%s  at  %s --- %s" % (
+        ws_namespace, ws_name, firecloud_api.PROD_API_ROOT, workspace_request.text))
+
+    # get the submission
+    submission_request = firecloud_api.get_submission(ws_namespace, ws_name, submission_id)
+    if submission_request.status_code != 200:
+        fail("Unable to find submission: %s" % submission_id)
+    submission_json = submission_request.json()
+
+    # translate the submission entity into workflow entity type
+    submission_entity_json = submission_json["submissionEntity"]
+    submission_entity_type = submission_entity_json["entityType"]
+    # either the submission entity type will be the same as the workflow entity type 
+    # (when it's a submission on a single entity) or it will be a set of those entities
+    # - if it's a set, just strip _set from the end to get the type the set contains
+    workflow_entity_type = submission_entity_type.replace("_set", "")
+
+    # create an update TSV with the attributes to bind back to the data model from this
+    # given submission's outputs
+    tsv_filename = 'entity_update.tsv'
+    with open(tsv_filename, 'wb') as entity_update_tsv:
+        entity_writer = csv.writer(entity_update_tsv, delimiter='\t')
+
+        # id column name is entityType_id
+        entity_id_column = "%s_id" % workflow_entity_type
+        # initial headers need to be update:entityType_id
+        tsv_headers = ['update:%s' % entity_id_column]
+
+        # get the method config used by this submission
+        mc_namespace = submission_json["methodConfigurationNamespace"]
+        mc_name = submission_json["methodConfigurationName"]
+        method_config_json = firecloud_api.get_workspace_config(ws_namespace, ws_name, mc_namespace, mc_name).json()
+
+        # go through each output and create a header for the attribute that is needed
+        mc_outputs = method_config_json["outputs"]
+        mc_output_keys = mc_outputs.keys()
+        for mc_output_key in mc_output_keys:
+            # if the user provided expression to override for this output then use that instead
+            if expressions_override and mc_output_key in expressions_override:
+                output_expression = expressions_override[mc_output_key]
+            # otherwise get the output expression from the method config
+            else:
+                output_expression = mc_outputs[mc_output_key]
+            output_attribute = output_expression.replace("this.", "")
+            tsv_headers.append(output_attribute)
+        entity_writer.writerow(tsv_headers)
+
+        # go through each workflow in this submission
+        submission_workflows_json = submission_json["workflows"]
+        succeeded_workflows = [w for w in submission_workflows_json if w["status"] == "Succeeded"]
+
+        num_workflows = len(succeeded_workflows)
+        for workflow_idx, workflow_json in enumerate(succeeded_workflows):
+            workflow_id = workflow_json["workflowId"]
+
+            print "Processing workflow %d of %d: %s" % (workflow_idx + 1, num_workflows, workflow_id)
+            entity_attribute_updates = []
+
+            workflow_entity_name = workflow_json["workflowEntity"]["entityName"]
+            # the first column needs to be the name of the entity
+            entity_attribute_updates.append(workflow_entity_name)
+
+            # get workflow metadata and outputs that were produced
+            workflow_metadata_json = get_workflow_metadata(ws_namespace, ws_name, submission_id, workflow_id)
+            workflow_outputs_json = workflow_metadata_json["outputs"]
+
+            # go through each method config output in the same order as the headers
+            for mc_output_key in mc_output_keys:
+                workflow_output = workflow_outputs_json[mc_output_key]
+
+                # add the value from this workflow output to the same column as the attribute to bind it to
+                entity_attribute_updates.append(workflow_output)
+
+            # write the row values for this entity
+            entity_writer.writerow(entity_attribute_updates)
+
+    upload = prompt_to_continue("Update TSV file has been produced.  Would you like to upload this file?")
+
+    if upload:
+        print "Uploading updated entities TSV..."
+        upload_request = firecloud_api.upload_entities_tsv(ws_namespace, ws_name, tsv_filename)
+
+        if upload_request.status_code != 200:
+            print "Error uploading updated entities TSV:", upload_request.text
+
+        print "Upload complete."
+        os.remove(tsv_filename)
+    else:
+        print "The file can be reviewed and manually uploaded - see %s" % tsv_filename
+
+
+def main():
+    setup()
+
+    # The main argument parser
+    parser = DefaultArgsParser(description=
+                               "Use an existing submission from a workspace to bind attributes back to the data model.  "
+                               "This can be used to fix issues with binding that may have occurred, or to revert outputs "
+                               "to a previous submission.  Additionally, an optional expression override can be used to "
+                               "provide a new output expreession for a given output (e.g. to bind an output attribute that "
+                               "was not originally bound.")
+
+    # Core application arguments
+    parser.add_argument('-p', '--workspace_namespace', dest='ws_namespace', action='store', required=True,
+                        help='Workspace namespace')
+    parser.add_argument('-n', '--workspace_name', dest='ws_name', action='store', required=True, help='Workspace name')
+    parser.add_argument('-s', '--submission_id', dest='submission_id', action='store', required=True,
+                        help='Submission Id')
+    parser.add_argument('-e', '--expressions_override', dest='expressions_override', action='store', required=False,
+                        help='Optional argument to override output expressions used in the method config for this submission.  Syntax is in the form \'{"output_name": "expression"}\'')
+
+    # Call the appropriate function for the given subcommand, passing in the parsed program arguments
+    args = parser.parse_args()
+
+    print "Note -- this script has the following limitations:"
+    print "        *  The output expressions must all be of the form 'this.attribute_name' - this does not handle " \
+          "           cases such as 'this.case_sample.attribute_name'"
+    print "        *  The root entity type chosen has to be either a single entity of root entity type or a set of " \
+          "           those entities.  This will not work for instance if your method runs on a sample and you " \
+          "           chose a pair with the launch expression 'this.case_sample'."
+    print "        *  The method config used for this submission can not have been deleted."
+
+    continue_script = prompt_to_continue("continue?")
+
+    expression_override = json.loads(args.expressions_override) if args.expressions_override else None
+    if continue_script:
+        fix_outputs(args.ws_namespace, args.ws_name, args.submission_id, expression_override)
+    else:
+        print "Exiting."
+
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file