clearwing

changeset 217:a22012e75b92 1.2.3-genius

Updating Code and configuration to match Firethorn version 2.1.25
author Stelios <stv@roe.ac.uk>
date Wed Jul 25 18:49:02 2018 +0300 (2018-07-25)
parents ff07df7a7c1d
children f9c379778b4d
files src/freeform_sql/firethorn_config.py src/freeform_sql/firethorn_helpers.py
line diff
     1.1 --- a/src/freeform_sql/firethorn_config.py	Wed Jul 25 18:17:44 2018 +0300
     1.2 +++ b/src/freeform_sql/firethorn_config.py	Wed Jul 25 18:49:02 2018 +0300
     1.3 @@ -8,56 +8,72 @@
     1.4  from config import base_url as webpy_gui_url
     1.5  from config import firethorn_base  as web_services_url
     1.6  from config import firethorn_tap_base
     1.7 -from config import FULL_SURVEYDBS, PRIVATE_SURVEYDBS_VPHAS
     1.8 -from config import PRIVATE_FULL_SURVEYDBS
     1.9  
    1.10 -### Directory Information (Do not change unless static directory is modified)
    1.11 -base_location = os.getcwd()
    1.12 -vospace_dir = base_location + '/static/static_vospace'
    1.13 -host_temp_directory = webpy_gui_url + 'static/static_vo_tool/temp/'
    1.14 -vospace_root = "/"
    1.15 -base_url = webpy_gui_url
    1.16 +sql_rowlimit=100000
    1.17 +sql_timeout = 1000
    1.18 +firethorn_timeout = 6000000
    1.19  
    1.20 +test_email = "test@test.roe.ac.uk"
    1.21  
    1.22 -### General globals
    1.23 -mode_global = 'async'
    1.24 -web.config.smtp_server = 'mail.roe.ac.uk'
    1.25 -from_email = 'osa-support@roe.ac.uk'
    1.26  
    1.27 +### Query Runtime and Polling Configurations ###
    1.28 +MAX_FILE_SIZE = 248576000 
    1.29 +delay = 3
    1.30 +MIN_ELAPSED_TIME_BEFORE_REDUCE = 40
    1.31 +MAX_ELAPSED_TIME = 18000
    1.32 +MAX_DELAY = 15
    1.33 +INITIAL_DELAY = 2
    1.34  
    1.35 +#firethorn.limits.rows.default=1000,0000
    1.36 +firethorn_limits_rows_default=sql_rowlimit
    1.37 +firethorn_limits_cells_default=0
    1.38 +firethorn_limits_time_default=0
    1.39 +
    1.40 +adql_query_delay = "adql.query.delay.first"
    1.41 +
    1.42 +#firethorn.limits.rows.absolute=1000000
    1.43 +firethorn_limits_rows_absolute=sql_rowlimit
    1.44 +firethorn_limits_cells_absolute=0
    1.45 +firethorn_limits_time_absolute=60000000
    1.46 +
    1.47 +
    1.48 +firethorn_limits_time = 6000000
    1.49  
    1.50  ### URL, Type and Parameter associations and Information
    1.51 -local_hostname = {
    1.52 -                  'index' : webpy_gui_url ,'services' : webpy_gui_url + '/services', 'jdbc_resources' : webpy_gui_url + '/jdbc_resources',
    1.53 -                  'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json' : webpy_gui_url + '/services',
    1.54 -                  'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-service-1.0.json' : webpy_gui_url + '/services',
    1.55 -                  'http://data.metagrid.co.uk/wfau/firethorn/types/entity/jdbc-resource-1.0.json' : webpy_gui_url +'/jdbc_resources',
    1.56 -                  'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-catalog-1.0.json' : webpy_gui_url + '/services',
    1.57 -                  'http://data.metagrid.co.uk/wfau/firethorn/types/entity/jdbc-catalog-1.0.json' : webpy_gui_url +'/jdbc_resources',
    1.58 -                  'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-schema-1.0.json' : webpy_gui_url + '/services',
    1.59 -                  'http://data.metagrid.co.uk/wfau/firethorn/types/entity/jdbc-schema-1.0.json' : webpy_gui_url +'/jdbc_resources',
    1.60 -                  'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-table-1.0.json' : webpy_gui_url + '/services',
    1.61 -                  'http://data.metagrid.co.uk/wfau/firethorn/types/entity/jdbc-table-1.0.json' : webpy_gui_url +'/jdbc_resources',
    1.62 -                  'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-column-1.0.json' : webpy_gui_url + '/services',
    1.63 -                  'http://data.metagrid.co.uk/wfau/firethorn/types/entity/jdbc-column-1.0.json' : webpy_gui_url +'/jdbc_resources'
    1.64 -                  }
    1.65 -
    1.66 -
    1.67  get_jdbc_resources_url = "/firethorn/jdbc/resource/select"
    1.68 -get_adql_resources_url = "/firethorn/adql/resource/select"
    1.69 +get_adql_resources_url = "/adql/resource/select"
    1.70 +get_ivoa_resources_url = "/ivoa/resource/select"
    1.71  
    1.72  get_param = 'id'
    1.73  
    1.74 -workspace_import_schema_name = "adql.resource.schema.import.name"
    1.75 -workspace_import_schema_base = "adql.resource.schema.import.base"
    1.76 +system_info = "/system/info"
    1.77 +
    1.78 +workspace_import_schema_name = "adql.schema.name"
    1.79 +workspace_import_schema_base = "adql.schema.base"
    1.80  workspace_import_uri = "/schemas/import"
    1.81  
    1.82  schema_import_schema_name = "adql.schema.table.import.name"
    1.83 -schema_import_schema_base = "adql.schema.table.import.base"
    1.84 +schema_import_schema_base = "adql.table.base"
    1.85  schema_import_uri = "/tables/import"
    1.86  
    1.87 +ivoa_schema_select_by_name_param = "ivoa.schema.name"
    1.88 +ivoa_table_select_by_name_param = "ivoa.table.name"
    1.89 +ivoa_column_select_by_name_param = "ivoa.column.name"
    1.90 +
    1.91 +jdbc_schema_catalog = "jdbc.schema.catalog"
    1.92 +jdbc_schema_schema = "jdbc.schema.schema"
    1.93 +jdbc_schema_ident = "jdbc.schema.ident"
    1.94 +
    1.95  query_create_uri = "/queries/create"
    1.96 +query_update_uri = "/queries/update"
    1.97  query_name_param = "adql.query.name"
    1.98 +
    1.99 +query_limit_rows_param = "adql.query.limit.rows"
   1.100 +query_limit_time_param = "adql.query.limit.time"
   1.101 +query_wait_time_param = "adql.query.wait.time"
   1.102 +
   1.103 +query_mode_param = "adql.query.mode"
   1.104 +
   1.105  query_param = "adql.query.input"
   1.106  query_status_update = "adql.query.status.next"
   1.107  
   1.108 @@ -66,22 +82,21 @@
   1.109  
   1.110  table_import_uri = '/tables/import'
   1.111  
   1.112 -workspace_creator = web_services_url + "/adql/resource/create"
   1.113 -
   1.114 -
   1.115 -create_menu_items = {'admin' : ['TAP Connection','JDBC connection', 'Workspace'] , 'user' : ['TAP Connection', 'Workspace']}
   1.116 +workspace_creator = "/adql/resource/create"
   1.117 +jdbc_creator = "/jdbc/resource/create"
   1.118 +ivoa_resource_create = "/ivoa/resource/create"
   1.119  
   1.120  
   1.121  resource_create_name_params = {
   1.122 -                               'http://data.metagrid.co.uk/wfau/firethorn/types/entity/jdbc-resource-1.0.json' : 'jdbc.resource.name',
   1.123 -                               'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json' : 'adql.resource.create.name',
   1.124 -                               'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-service-1.0.json' : 'adql.resource.create.name',
   1.125 +                               'http://data.metagrid.co.uk/wfau/firethorn/types/entity/jdbc-resource-1.0.json' : 'jdbc.resource.name', 
   1.126 +                               'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json' : 'adql.resource.name',
   1.127 +                               'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-service-1.0.json' : 'adql.resource.name',
   1.128                                 'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-schema-1.0.json' : 'adql.schema.name'
   1.129                                 }
   1.130  
   1.131  
   1.132  resource_create_url_params = {
   1.133 -                              'http://data.metagrid.co.uk/wfau/firethorn/types/entity/jdbc-resource-1.0.json' : 'jdbc.connection.url',
   1.134 +                              'http://data.metagrid.co.uk/wfau/firethorn/types/entity/jdbc-resource-1.0.json' : 'jdbc.resource.create.url',
   1.135                                'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json' : 'adql.resource.create.url',
   1.136                                'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-service-1.0.json' : 'adql.resource.create.url'
   1.137  
   1.138 @@ -89,9 +104,9 @@
   1.139  
   1.140  
   1.141  resource_create_username_params = {
   1.142 -                                   'http://data.metagrid.co.uk/wfau/firethorn/types/entity/jdbc-resource-1.0.json' : 'jdbc.connection.user',
   1.143 -                                   'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json' : 'adql.resource.create.user',
   1.144 -                                   'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-service-1.0.json' : 'adql.resource.create.user'
   1.145 +                                   'http://data.metagrid.co.uk/wfau/firethorn/types/entity/jdbc-resource-1.0.json' : 'jdbc.connection.identity',
   1.146 +                                   'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json' : 'adql.resource.create.identity',
   1.147 +                                   'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-service-1.0.json' : 'adql.resource.create.identity'
   1.148                                     }
   1.149  
   1.150  
   1.151 @@ -103,38 +118,38 @@
   1.152  
   1.153  
   1.154  create_urls = {
   1.155 -               'http://data.metagrid.co.uk/wfau/firethorn/types/entity/jdbc-resource-1.0.json' : web_services_url + '/firethorn/jdbc/resource/create',
   1.156 -               'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json' : web_services_url + '/firethorn/adql/resource/create',
   1.157 -               'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-service-1.0.json' : web_services_url + '/firethorn/adql/resource/create'
   1.158 +               'http://data.metagrid.co.uk/wfau/firethorn/types/entity/jdbc-resource-1.0.json' : '/firethorn/jdbc/resource/create', 
   1.159 +               'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json' : '/firethorn/adql/resource/create',
   1.160 +               'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-service-1.0.json' : '/firethorn/adql/resource/create'
   1.161                 }
   1.162  
   1.163  
   1.164  get_urls = {
   1.165 -            'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-service-1.0.json' : web_services_url + '/firethorn/adql/resource/',
   1.166 -            'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json' : web_services_url + '/firethorn/adql/resource/',
   1.167 -            'http://data.metagrid.co.uk/wfau/firethorn/types/entity/jdbc-resource-1.0.json' : web_services_url + '/firethorn/jdbc/resource/'
   1.168 +            'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-service-1.0.json' : '/firethorn/adql/resource/',
   1.169 +            'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json' : '/firethorn/adql/resource/',
   1.170 +            'http://data.metagrid.co.uk/wfau/firethorn/types/entity/jdbc-resource-1.0.json' : '/firethorn/jdbc/resource/'
   1.171              }
   1.172  
   1.173  
   1.174  db_select_by_name_urls = {
   1.175 -                          'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-service-1.0.json' : web_services_url + '/firethorn/adql/resource/select?',
   1.176 -                          'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json' : web_services_url + '/firethorn/adql/resource/select?',
   1.177 -                          'http://data.metagrid.co.uk/wfau/firethorn/types/entity/jdbc-resource-1.0.json' : web_services_url + '/firethorn/jdbc/resource/select?'
   1.178 +                          'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-service-1.0.json' : '/firethorn/adql/resource/select?',
   1.179 +                          'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json' : '/firethorn/adql/resource/select?',
   1.180 +                          'http://data.metagrid.co.uk/wfau/firethorn/types/entity/jdbc-resource-1.0.json' : '/firethorn/jdbc/resource/select?'
   1.181                            }
   1.182  
   1.183  
   1.184  db_select_with_text_urls = {
   1.185 -                            'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-service-1.0.json' :  web_services_url + '/firethorn/adql/resource/search?',
   1.186 -                            'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json' :  web_services_url + '/firethorn/adql/resource/search?',
   1.187 -                            'http://data.metagrid.co.uk/wfau/firethorn/types/entity/jdbc-resource-1.0.json' :  web_services_url + ' /firethorn/jdbc/resource/search?'
   1.188 +                            'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-service-1.0.json' :  '/firethorn/adql/resource/search?',
   1.189 +                            'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json' :  '/firethorn/adql/resource/search?',
   1.190 +                            'http://data.metagrid.co.uk/wfau/firethorn/types/entity/jdbc-resource-1.0.json' :  ' /firethorn/jdbc/resource/search?'
   1.191                              }
   1.192  
   1.193  type_select_uris = {'schemas' : '/schemas/select',
   1.194                      'tables' : '/tables/select',
   1.195                      'columns' : '/columns/select',
   1.196                      'workspaces' :'firethorn/adql/resources/select'
   1.197 -                    }
   1.198 -
   1.199 +                    }                   
   1.200 +                       
   1.201  resource_uris = {
   1.202                   'http://data.metagrid.co.uk/wfau/firethorn/types/entity/jdbc-resource-1.0.json': '/schemas/select',
   1.203                   'http://data.metagrid.co.uk/wfau/firethorn/types/entity/jdbc-catalog-1.0.json': '/schemas/select',
   1.204 @@ -175,17 +190,16 @@
   1.205                              'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json': 'adql.resource.select.name',
   1.206                              'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-service-1.0.json': 'adql.resource.select.name'
   1.207                              }
   1.208 -schema_select_by_name_param = "adql.resource.schema.select.name"
   1.209 -table_select_by_name_param = "adql.schema.table.select.name"
   1.210 +
   1.211  
   1.212  types = {
   1.213 -         'service' : 'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-service-1.0.json',
   1.214 +         'service' : 'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-service-1.0.json', 
   1.215           'Service' : 'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-service-1.0.json',
   1.216           'JDBC connection' : 'http://data.metagrid.co.uk/wfau/firethorn/types/entity/jdbc-resource-1.0.json',
   1.217           'resource' : 'http://data.metagrid.co.uk/wfau/firethorn/types/entity/jdbc-resource-1.0.json',
   1.218 -         'catalog' : 'http://data.metagrid.co.uk/wfau/firethorn/types/entity/jdbc-catalog-1.0.json',
   1.219 -         'jdbc_catalog' : 'http://data.metagrid.co.uk/wfau/firethorn/types/entity/jdbc-catalog-1.0.json',
   1.220 -         'adql_catalog' : 'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-catalog-1.0.json',
   1.221 +         'catalog' : 'http://data.metagrid.co.uk/wfau/firethorn/types/entity/jdbc-catalog-1.0.json', 
   1.222 +         'jdbc_catalog' : 'http://data.metagrid.co.uk/wfau/firethorn/types/entity/jdbc-catalog-1.0.json', 
   1.223 +         'adql_catalog' : 'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-catalog-1.0.json', 
   1.224           'jdbc_table' : 'http://data.metagrid.co.uk/wfau/firethorn/types/entity/jdbc-table-1.0.json',
   1.225           'adql_table' : 'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-table-1.0.json',
   1.226           'jdbc_schema' : 'http://data.metagrid.co.uk/wfau/firethorn/types/entity/jdbc-schema-1.0.json',
     2.1 --- a/src/freeform_sql/firethorn_helpers.py	Wed Jul 25 18:17:44 2018 +0300
     2.2 +++ b/src/freeform_sql/firethorn_helpers.py	Wed Jul 25 18:49:02 2018 +0300
     2.3 @@ -30,11 +30,10 @@
     2.4  
     2.5  
     2.6          if (session.get("community_input","public (unknown)").lower()!="vphas"):
     2.7 -            if not ((i["fullname"].lower() in (name.lower() for name in firethorn_config.PRIVATE_SURVEYDBS_VPHAS))):
     2.8 -                if exclude_schema!=i["self"] and exclude_schema_name.lower()!=i["fullname"].lower():
     2.9 -                    available_list += """
    2.10 -                        <li><span>""" + i["fullname"] + """</span> as <input type="text" name="catalogue" value='""" +  i["name"]  + """' /><input type="hidden" name="ident" value='""" + i["self"] +"""'/>""" + add_catalogue_html + """</li>
    2.11 -                    """
    2.12 +            if exclude_schema!=i["self"] and exclude_schema_name.lower()!=i["fullname"].lower():
    2.13 +                available_list += """
    2.14 +                    <li><span>""" + i["fullname"] + """</span> as <input type="text" name="catalogue" value='""" +  i["name"]  + """' /><input type="hidden" name="ident" value='""" + i["self"] +"""'/>""" + add_catalogue_html + """</li>
    2.15 +                """
    2.16          else:
    2.17  
    2.18              if not ((i["fullname"].lower() in (name.lower() for name in firethorn_config.PRIVATE_FULL_SURVEYDBS))  and (log_info == 'False' or log_info == False)):
    2.19 @@ -65,7 +64,7 @@
    2.20          ### Create workspace
    2.21          workspace_name = 'workspace-' + t.strftime("%y%m%d_%H%M%S")
    2.22          data = urllib.urlencode({firethorn_config.resource_create_name_params['http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json'] : workspace_name})
    2.23 -        req = urllib2.Request( firethorn_config.workspace_creator, data, headers={"Accept" : "application/json", "firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : config.firethorn_admin_community, "firethorn.auth.username" : config.firethorn_admin_username, "firethorn.auth.password" : config.firethorn_admin_password})
    2.24 +        req = urllib2.Request(config.firethorn_base + firethorn_config.workspace_creator, data, headers={"Accept" : "application/json", "firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : config.firethorn_admin_community, "firethorn.auth.username" : config.firethorn_admin_username, "firethorn.auth.password" : config.firethorn_admin_password})
    2.25  
    2.26          response = urllib2.urlopen(req)
    2.27          js = json.loads(response.read())
    2.28 @@ -143,7 +142,7 @@
    2.29  
    2.30      try:
    2.31        
    2.32 -        workspace = firethorn_config.firethorn_tap_base
    2.33 +        workspace = config.firethorn_tap_base
    2.34  
    2.35          
    2.36          if keyword_type.lower()=="schema":