clearwing

changeset 174:f2b49096e447 1.2.3-genius

Changes for BlueQuery
author Stelios <stv@roe.ac.uk>
date Sat Sep 10 15:54:13 2016 +0300 (2016-09-10)
parents c995092e2ded
children c01ce6c825bc
files src/freeform_sql/firethorn_config.py src/freeform_sql/misc.py
line diff
     1.1 --- a/src/freeform_sql/firethorn_config.py	Sun Apr 03 15:52:16 2016 +0300
     1.2 +++ b/src/freeform_sql/firethorn_config.py	Sat Sep 10 15:54:13 2016 +0300
     1.3 @@ -57,9 +57,9 @@
     1.4  schema_import_uri = "/tables/import"
     1.5  
     1.6  query_create_uri = "/queries/create"
     1.7 -query_name_param = "adql.schema.query.create.name"
     1.8 -query_param = "adql.schema.query.create.query"
     1.9 -query_status_update = "adql.query.update.status"
    1.10 +query_name_param = "adql.query.name"
    1.11 +query_param = "adql.query.input"
    1.12 +query_status_update = "adql.query.status.next"
    1.13  
    1.14  schema_create_uri = '/schemas/create'
    1.15  table_create_uri = '/tables/create'
    1.16 @@ -73,15 +73,15 @@
    1.17  
    1.18  
    1.19  resource_create_name_params = {
    1.20 -                               'http://data.metagrid.co.uk/wfau/firethorn/types/entity/jdbc-resource-1.0.json' : 'jdbc.resource.create.name',
    1.21 +                               'http://data.metagrid.co.uk/wfau/firethorn/types/entity/jdbc-resource-1.0.json' : 'jdbc.resource.name',
    1.22                                 'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json' : 'adql.resource.create.name',
    1.23                                 'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-service-1.0.json' : 'adql.resource.create.name',
    1.24 -                               'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-schema-1.0.json' : 'adql.resource.schema.create.name'
    1.25 +                               'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-schema-1.0.json' : 'adql.schema.name'
    1.26                                 }
    1.27  
    1.28  
    1.29  resource_create_url_params = {
    1.30 -                              'http://data.metagrid.co.uk/wfau/firethorn/types/entity/jdbc-resource-1.0.json' : 'jdbc.resource.create.url',
    1.31 +                              'http://data.metagrid.co.uk/wfau/firethorn/types/entity/jdbc-resource-1.0.json' : 'jdbc.connection.url',
    1.32                                'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json' : 'adql.resource.create.url',
    1.33                                'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-service-1.0.json' : 'adql.resource.create.url'
    1.34  
    1.35 @@ -89,14 +89,14 @@
    1.36  
    1.37  
    1.38  resource_create_username_params = {
    1.39 -                                   'http://data.metagrid.co.uk/wfau/firethorn/types/entity/jdbc-resource-1.0.json' : 'jdbc.resource.create.user',
    1.40 +                                   'http://data.metagrid.co.uk/wfau/firethorn/types/entity/jdbc-resource-1.0.json' : 'jdbc.connection.user',
    1.41                                     'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json' : 'adql.resource.create.user',
    1.42                                     'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-service-1.0.json' : 'adql.resource.create.user'
    1.43                                     }
    1.44  
    1.45  
    1.46  resource_create_password_params = {
    1.47 -                                   'http://data.metagrid.co.uk/wfau/firethorn/types/entity/jdbc-resource-1.0.json' : 'jdbc.resource.create.pass',
    1.48 +                                   'http://data.metagrid.co.uk/wfau/firethorn/types/entity/jdbc-resource-1.0.json' : 'jdbc.connection.pass',
    1.49                                     'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json' : 'adql.resource.create.pass',
    1.50                                     'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-service-1.0.json' : 'adql.resource.create.pass'
    1.51                                     }
    1.52 @@ -171,7 +171,7 @@
    1.53  
    1.54  
    1.55  db_select_by_name_params = {
    1.56 -                            'http://data.metagrid.co.uk/wfau/firethorn/types/entity/jdbc-resource-1.0.json': 'jdbc.resource.select.name',
    1.57 +                            'http://data.metagrid.co.uk/wfau/firethorn/types/entity/jdbc-resource-1.0.json': 'jdbc.resource.name',
    1.58                              'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json': 'adql.resource.select.name',
    1.59                              'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-service-1.0.json': 'adql.resource.select.name'
    1.60                              }
     2.1 --- a/src/freeform_sql/misc.py	Sun Apr 03 15:52:16 2016 +0300
     2.2 +++ b/src/freeform_sql/misc.py	Sat Sep 10 15:54:13 2016 +0300
     2.3 @@ -403,19 +403,23 @@
     2.4      try:
     2.5          logging.exception("Started Firethorn job :::" +  strftime("%Y-%m-%d %H:%M:%S", gmtime()))
     2.6  
     2.7 -        data = urllib.urlencode({ firethorn_config.query_status_update : "RUNNING"})
     2.8 +        data = urllib.urlencode({ firethorn_config.query_status_update : "COMPLETED", "adql.query.wait.time" : 60000})
     2.9          
    2.10 -        #data = urllib.urlencode({ firethorn_config.query_status_update : "RUNNING", 'adql.query.update.delay.every' : '10000', 'adql.query.update.delay.first':'10000', 'adql.query.update.delay.last':'10000'})
    2.11 +        #data = urllib.urlencode({ firethorn_config.query_status_update : "RUNNING", 'adql.query.delay.every' : '10000', 'adql.query.delay.first':'10000', 'adql.query.delay.last':'10000'})
    2.12          request = urllib2.Request(url, data, headers={"Accept" : "application/json", "firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : session.get("community_input","public (unknown)")})
    2.13 +        logging.debug(url)
    2.14 +        logging.debug(data)
    2.15          f_update = urllib2.urlopen(request)
    2.16          query_json =  json.loads(f_update.read())
    2.17 -        query_status = query_json["status"]
    2.18 +        query_status = "QUEUED"
    2.19          
    2.20          logging.exception("Query started on Firethorn:")
    2.21          logging.exception(url)
    2.22     
    2.23 -        while query_status=="PENDING" or query_status=="RUNNING" and elapsed_time<MAX_ELAPSED_TIME:
    2.24 +        logging.debug(query_status)
    2.25 +        while query_status=="QUEUED" or query_status=="RUNNING" or query_status=="READY" and elapsed_time<MAX_ELAPSED_TIME:
    2.26              query_json = json.loads(get_status(url))
    2.27 +            logging.debug(get_status(url))
    2.28              query_status= query_json["status"] 
    2.29              time.sleep(delay)
    2.30              if elapsed_time>MIN_ELAPSED_TIME_BEFORE_REDUCE and delay<MAX_DELAY:
    2.31 @@ -431,7 +435,7 @@
    2.32              logging.exception( "Editing status in start_query_loop:" )
    2.33              return {'Code' :-1,  'Content' : 'Query error: ' + query_json["syntax"]["status"] + ' - ' + query_json["syntax"]["friendly"] }
    2.34          elif query_status=="COMPLETED":
    2.35 -            return {'Code' :1,  'Content' : query_json["results"]["datatable"] }
    2.36 +            return {'Code' :1,  'Content' : query_json["results"]["formats"]["datatable"] }
    2.37          elif elapsed_time>=MAX_ELAPSED_TIME:
    2.38              return {'Code' :-1,  'Content' : 'Query error: Max run time (' + str(MAX_ELAPSED_TIME) + ' seconds) exceeded' }
    2.39          else:
    2.40 @@ -680,8 +684,14 @@
    2.41          if query_name=="":
    2.42              query_name = 'query-' + t.strftime("%y%m%d_%H%M%S")
    2.43       
    2.44 -        #data = urllib.urlencode({ firethorn_config.query_name_param : query_name,  firethorn_config.query_param : query,'adql.query.update.delay.every' : '10000', 'adql.query.update.delay.first':'10000', 'adql.query.update.delay.last':'10000'})
    2.45 -        
    2.46 +        #data = urllib.urlencode({ firethorn_config.query_name_param : query_name,  firethorn_config.query_param : query,'adql.query.delay.every' : '10000', 'adql.query.delay.first':'10000', 'adql.query.delay.last':'10000'})
    2.47 +        def read_json(url):
    2.48 +            request2 = urllib2.Request(url, headers={"Accept" : "application/json", "firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : "public (unknown)"})
    2.49 +            f_read = urllib2.urlopen(request2)
    2.50 +            query_json = f_read.read()
    2.51 +            f_read.close()
    2.52 +            return query_json   
    2.53 +
    2.54          urlenc = { firethorn_config.query_name_param : query_name,  firethorn_config.query_param : query}
    2.55          data = urllib.urlencode(urlenc)
    2.56          request = urllib2.Request(query_space + firethorn_config.query_create_uri, data, headers={"Accept" : "application/json", "firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : session.get("community_input","public (unknown)")})
    2.57 @@ -690,7 +700,7 @@
    2.58          query_create_result = json.loads(f.read())
    2.59          query_identity = query_create_result["ident"]
    2.60          query_loop_results = start_query_loop(query_identity)
    2.61 -        results_adql_url = query_create_result["results"]["adql"]
    2.62 +        results_adql_url = json.loads(read_json(query_identity)).get("results",None).get("table",None)
    2.63  
    2.64          
    2.65          if query_loop_results.get("Code", "") !="":