clearwing

changeset 219:90ea0520f7e5 1.2.3-genius

Modifying code to match Firethorn version 2.1.25 API continued..
author Stelios <stv@roe.ac.uk>
date Thu Jul 26 16:26:39 2018 +0300 (2018-07-26)
parents f9c379778b4d
children b8cd53f93386
files src/freeform_sql/misc.py src/url_classes/data_tables_processing.py
line diff
     1.1 --- a/src/freeform_sql/misc.py	Wed Jul 25 18:54:32 2018 +0300
     1.2 +++ b/src/freeform_sql/misc.py	Thu Jul 26 16:26:39 2018 +0300
     1.3 @@ -185,7 +185,7 @@
     1.4          ### Create workspace
     1.5          workspace_name = 'workspace-' + t.strftime("%y%m%d_%H%M%S") 
     1.6          data = urllib.urlencode({firethorn_config.resource_create_name_params['http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json'] : workspace_name})
     1.7 -        req = urllib2.Request( config.firethorn_base + firethorn_config.workspace_creator, data, headers={"Accept" : "application/json", "firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : config.firethorn_admin_community, "firethorn.auth.username" : config.firethorn_admin_username, "firethorn.auth.password" : config.firethorn_admin_password})
     1.8 +        req = urllib2.Request(config.firethorn_base + firethorn_config.workspace_creator, data, headers={"Accept" : "application/json", "firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : config.firethorn_admin_community, "firethorn.auth.username" : config.firethorn_admin_username, "firethorn.auth.password" : config.firethorn_admin_password})
     1.9          response = urllib2.urlopen(req)
    1.10          workspace = json.loads(response.read())["self"]
    1.11          response.close()
    1.12 @@ -390,6 +390,8 @@
    1.13          request2 = urllib2.Request(url, headers={"Accept" : "application/json", "firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : config.firethorn_admin_community, "firethorn.auth.username" : config.firethorn_admin_username, "firethorn.auth.password" : config.firethorn_admin_password})
    1.14          f_read = urllib2.urlopen(request2)
    1.15          query_json = f_read.read()
    1.16 +        query_status = json.loads(query_json)["status"]
    1.17 +
    1.18          return query_json
    1.19      
    1.20      max_size_exceeded = False 
    1.21 @@ -400,27 +402,16 @@
    1.22      start_time = time.time()
    1.23      elapsed_time = 0
    1.24      query_json = {'syntax' : {'friendly' : 'A problem occurred while running your query', 'status' : 'Error' }}
    1.25 +    query_status = "READY" 
    1.26      
    1.27      try:
    1.28          logging.exception("Started Firethorn job :::" +  strftime("%Y-%m-%d %H:%M:%S", gmtime()))
    1.29  
    1.30 -        data = urllib.urlencode({ firethorn_config.query_status_update : "COMPLETED", "adql.query.wait.time" : 60000})
    1.31 -        
    1.32 -        #data = urllib.urlencode({ firethorn_config.query_status_update : "RUNNING", 'adql.query.delay.every' : '10000', 'adql.query.delay.first':'10000', 'adql.query.delay.last':'10000'})
    1.33 -        request = urllib2.Request(url, data, headers={"Accept" : "application/json", "firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : config.firethorn_admin_community, "firethorn.auth.username" : config.firethorn_admin_username, "firethorn.auth.password" : config.firethorn_admin_password})
    1.34 -
    1.35 -
    1.36 -        f_update = urllib2.urlopen(request)
    1.37 -        query_json =  json.loads(f_update.read())
    1.38 -        query_status = "QUEUED"
    1.39 -        
    1.40 -        logging.exception("Query started on Firethorn:")
    1.41 -
    1.42 -   
    1.43 -
    1.44          while query_status=="QUEUED" or query_status=="RUNNING" or query_status=="READY" and elapsed_time<MAX_ELAPSED_TIME:
    1.45 -            query_json = json.loads(get_status(url))
    1.46 -            query_status= query_json["status"] 
    1.47 +            request = urllib2.Request(url, headers={"Accept" : "application/json", "firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : config.firethorn_admin_community, "firethorn.auth.username" : config.firethorn_admin_username, "firethorn.auth.password" : config.firethorn_admin_password})
    1.48 +            resp = urllib2.urlopen(request)
    1.49 +            query_json =  json.loads(resp.read())
    1.50 +            query_status =  query_json["status"]
    1.51              time.sleep(delay)
    1.52              if elapsed_time>MIN_ELAPSED_TIME_BEFORE_REDUCE and delay<MAX_DELAY:
    1.53                  delay = delay + delay
    1.54 @@ -690,15 +681,17 @@
    1.55       
    1.56          #data = urllib.urlencode({ firethorn_config.query_name_param : query_name,  firethorn_config.query_param : query,'adql.query.delay.every' : '10000', 'adql.query.delay.first':'10000', 'adql.query.delay.last':'10000'})
    1.57          def read_json(url):
    1.58 -            request2 = urllib2.Request(url, headers={"Accept" : "application/json", "firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : "public (unknown)"})
    1.59 +            request2 = urllib2.Request(url,headers={"Accept" : "application/json", "firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : config.firethorn_admin_community, "firethorn.auth.username" : config.firethorn_admin_username, "firethorn.auth.password" : config.firethorn_admin_password})
    1.60              f_read = urllib2.urlopen(request2)
    1.61              query_json = f_read.read()
    1.62              f_read.close()
    1.63              return query_json   
    1.64  
    1.65 -        urlenc = { firethorn_config.query_name_param : query_name,  firethorn_config.query_param : query}
    1.66 +        urlenc = { firethorn_config.query_name_param : query_name,  firethorn_config.query_param : query, firethorn_config.query_status_update : "COMPLETED"}
    1.67          data = urllib.urlencode(urlenc)
    1.68 -        request = urllib2.Request(query_space + firethorn_config.query_create_uri, data, headers={"Accept" : "application/json", "firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : config.firethorn_admin_community, "firethorn.auth.username" : config.firethorn_admin_username, "firethorn.auth.password" : config.firethorn_admin_password})
    1.69 +
    1.70 +
    1.71 +        request = urllib2.Request(json.loads(read_json(query_space))["parent"] + firethorn_config.query_create_uri, data, headers={"Accept" : "application/json", "firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : config.firethorn_admin_community, "firethorn.auth.username" : config.firethorn_admin_username, "firethorn.auth.password" : config.firethorn_admin_password})
    1.72  
    1.73          f = urllib2.urlopen(request)
    1.74          query_create_result = json.loads(f.read())
    1.75 @@ -729,7 +722,7 @@
    1.76              if query_loop_results.get("Code", "") ==-1:
    1.77                  result = query_loop_results.get("Content", "Error")
    1.78              elif query_loop_results.get("Code", "") ==1:
    1.79 -                req = urllib2.Request(query_loop_results.get("Content", ""), headers={"firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : session.get("community_input","public (unknown)")})
    1.80 +                req = urllib2.Request(query_loop_results.get("Content", ""),headers={"Accept" : "application/json", "firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : config.firethorn_admin_community, "firethorn.auth.username" : config.firethorn_admin_username, "firethorn.auth.password" : config.firethorn_admin_password} )
    1.81                  f = urllib2.urlopen(req)
    1.82                  datatable = f.read(MAX_FILE_SIZE) 
    1.83               
    1.84 @@ -1273,4 +1266,4 @@
    1.85      response = urllib2.urlopen(req)
    1.86      info = json.loads(response.read())
    1.87      response.close()
    1.88 -    return info
    1.89 +    return info
    1.90 \ No newline at end of file
     2.1 --- a/src/url_classes/data_tables_processing.py	Wed Jul 25 18:54:32 2018 +0300
     2.2 +++ b/src/url_classes/data_tables_processing.py	Thu Jul 26 16:26:39 2018 +0300
     2.3 @@ -1036,7 +1036,7 @@
     2.4                  queryvotable = query_json["results"]["formats"]["votable"]
     2.5  
     2.6                  web.header('Content-disposition', 'attachment; filename=download.xml')
     2.7 -                req = urllib2.Request( queryvotable, headers={ "firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : session.get("community_input","public (unknown)")} )
     2.8 +                req = urllib2.Request( queryvotable, headers={"Accept" : "application/json", "firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : config.firethorn_admin_community, "firethorn.auth.username" : config.firethorn_admin_username, "firethorn.auth.password" : config.firethorn_admin_password} )
     2.9                  response = urllib2.urlopen(req)
    2.10                  return_votable = response.read()  
    2.11                  response.close()