clearwing

changeset 207:f6908bbffe58

Changes to the parameter names for Firethorn version 2.1.10
author Stelios <stv@roe.ac.uk>
date Tue Jun 13 22:19:51 2017 +0300 (2017-06-13)
parents a8cadd426134
children e1e12d1f9048
files src/freeform_sql/firethorn_config.py src/freeform_sql/firethorn_helpers.py src/freeform_sql/misc.py src/url_classes/misc.py
line diff
     1.1 --- a/src/freeform_sql/firethorn_config.py	Fri May 26 11:59:12 2017 +0100
     1.2 +++ b/src/freeform_sql/firethorn_config.py	Tue Jun 13 22:19:51 2017 +0300
     1.3 @@ -16,13 +16,41 @@
     1.4  host_temp_directory = webpy_gui_url + 'static/static_vo_tool/temp/'
     1.5  vospace_root = "/"
     1.6  base_url = webpy_gui_url 
     1.7 -
     1.8 +sql_rowlimit=1000
     1.9  
    1.10  ### General globals
    1.11  mode_global = 'async'
    1.12  web.config.smtp_server = 'mail.roe.ac.uk'
    1.13  from_email = 'osa-support@roe.ac.uk'
    1.14 +    full_firethorn_host = "peter:8080"
    1.15 +    sql_rowlimit=100000
    1.16 +    sql_timeout = 1000
    1.17 +    firethorn_timeout = 6000000
    1.18  
    1.19 +web_services_url = "http://" + full_firethorn_host + "/firethorn"
    1.20 +web_services_sys_info = web_services_url + "/system/info"
    1.21 +tap_base = "http://" + full_firethorn_host + "/firethorn/tap/"
    1.22 +
    1.23 +### Query Runtime and Polling Configurations ###
    1.24 +MAX_FILE_SIZE = 248576000 
    1.25 +delay = 3
    1.26 +MIN_ELAPSED_TIME_BEFORE_REDUCE = 40
    1.27 +MAX_ELAPSED_TIME = 18000
    1.28 +MAX_DELAY = 15
    1.29 +INITIAL_DELAY = 2
    1.30 +
    1.31 +#firethorn.limits.rows.default=1000,0000
    1.32 +firethorn_limits_rows_default=sql_rowlimit
    1.33 +firethorn_limits_cells_default=0
    1.34 +firethorn_limits_time_default=0
    1.35 +
    1.36 +#firethorn.limits.rows.absolute=1000000
    1.37 +firethorn_limits_rows_absolute=sql_rowlimit
    1.38 +firethorn_limits_cells_absolute=0
    1.39 +firethorn_limits_time_absolute=60000000
    1.40 +
    1.41 +
    1.42 +firethorn_limits_time = 6000000
    1.43  
    1.44  
    1.45  ### URL, Type and Parameter associations and Information
    1.46 @@ -47,12 +75,12 @@
    1.47  
    1.48  get_param = 'id'
    1.49  
    1.50 -workspace_import_schema_name = "adql.resource.schema.import.name"
    1.51 -workspace_import_schema_base = "adql.resource.schema.import.base"
    1.52 +workspace_import_schema_name = "adql.schema.name"
    1.53 +workspace_import_schema_base = "adql.schema.base"
    1.54  workspace_import_uri = "/schemas/import"
    1.55  
    1.56  schema_import_schema_name = "adql.schema.table.import.name"
    1.57 -schema_import_schema_base = "adql.schema.table.import.base"
    1.58 +schema_import_schema_base = "adql.table.base"
    1.59  schema_import_uri = "/tables/import"
    1.60  
    1.61  query_create_uri = "/queries/create"
    1.62 @@ -73,8 +101,8 @@
    1.63  
    1.64  resource_create_name_params = {
    1.65                                 'http://data.metagrid.co.uk/wfau/firethorn/types/entity/jdbc-resource-1.0.json' : 'jdbc.resource.create.name', 
    1.66 -                               'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json' : 'adql.resource.create.name',
    1.67 -                               'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-service-1.0.json' : 'adql.resource.create.name',
    1.68 +                               'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json' : 'adql.resource.name',
    1.69 +                               'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-service-1.0.json' : 'adql.resource.name',
    1.70                                 'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-schema-1.0.json' : 'adql.resource.schema.create.name'
    1.71                                 }
    1.72  
    1.73 @@ -174,8 +202,8 @@
    1.74                              'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json': 'adql.resource.select.name',
    1.75                              'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-service-1.0.json': 'adql.resource.select.name'
    1.76                              }
    1.77 -schema_select_by_name_param = "adql.resource.schema.select.name"
    1.78 -table_select_by_name_param = "adql.schema.table.select.name"
    1.79 +schema_select_by_name_param = "adql.schema.name"
    1.80 +table_select_by_name_param = "adql.table.name"
    1.81  
    1.82  types = {
    1.83           'service' : 'http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-service-1.0.json', 
     2.1 --- a/src/freeform_sql/firethorn_helpers.py	Fri May 26 11:59:12 2017 +0100
     2.2 +++ b/src/freeform_sql/firethorn_helpers.py	Tue Jun 13 22:19:51 2017 +0300
     2.3 @@ -95,7 +95,7 @@
     2.4              importname = name
     2.5          
     2.6          if importname!="":
     2.7 -            data = urllib.urlencode({'urn:adql.copy.depth' : "THIN", firethorn_config.workspace_import_schema_base : ident, firethorn_config.workspace_import_schema_name : importname})
     2.8 +            data = urllib.urlencode({firethorn_config.workspace_import_schema_base : ident, firethorn_config.workspace_import_schema_name : importname})
     2.9        
    2.10          req = urllib2.Request( workspace + firethorn_config.workspace_import_uri, data, headers={"Accept" : "application/json", "firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : session.get("community_input","public (unknown)")}) 
    2.11          response = urllib2.urlopen(req)
     3.1 --- a/src/freeform_sql/misc.py	Fri May 26 11:59:12 2017 +0100
     3.2 +++ b/src/freeform_sql/misc.py	Tue Jun 13 22:19:51 2017 +0300
     3.3 @@ -210,9 +210,9 @@
     3.4                  importname = name
     3.5          
     3.6              if importname!="":
     3.7 -                data = urllib.urlencode({'urn:adql.copy.depth' : "THIN", firethorn_config.workspace_import_schema_base : ident, firethorn_config.workspace_import_schema_name : importname})
     3.8 +                data = urllib.urlencode({firethorn_config.workspace_import_schema_base : ident, firethorn_config.workspace_import_schema_name : importname})
     3.9              else :
    3.10 -                data = urllib.urlencode({'urn:adql.copy.depth' : "THIN", firethorn_config.workspace_import_schema_base : ident})
    3.11 +                data = urllib.urlencode({firethorn_config.workspace_import_schema_base : ident})
    3.12              
    3.13            
    3.14              req = urllib2.Request( workspace + firethorn_config.workspace_import_uri, data, headers={"Accept" : "application/json", "firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : session.get("community_input","public (unknown)")}) 
     4.1 --- a/src/url_classes/misc.py	Fri May 26 11:59:12 2017 +0100
     4.2 +++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
     4.3 @@ -1,1258 +0,0 @@
     4.4 -"""
     4.5 -misc module
     4.6 -
     4.7 -Miscellaneous functions used by for freeform SQL query processing for use with TAP services and XML VOTable procesing
     4.8 -
     4.9 -Created on Nov 30, 2011
    4.10 -@author: stelios
    4.11 -"""
    4.12 -
    4.13 -import os
    4.14 -import urllib2
    4.15 -import urllib
    4.16 -import StringIO
    4.17 -import time
    4.18 -import xml.dom.minidom
    4.19 -import atpy
    4.20 -import pyodbc
    4.21 -try:
    4.22 -    import simplejson as json
    4.23 -except ImportError:
    4.24 -    import json
    4.25 -import numpy
    4.26 -import html_functions
    4.27 -import xml_parser
    4.28 -from freeform_config import *
    4.29 -import re
    4.30 -from file_handler import File_handler
    4.31 -from helper_functions.string_functions import string_functions
    4.32 -string_functions = string_functions()
    4.33 -from config import survey_prefix, SURVEY_DB,base_location, debug_mode, MAX_DELAY, MAX_ELAPSED_TIME, MIN_ELAPSED_TIME_BEFORE_REDUCE, INITIAL_DELAY
    4.34 -from survey_globals import archive_input
    4.35 -from globals import MyOutputStream, logging
    4.36 -import datetime
    4.37 -import firethorn_config
    4.38 -from app import session
    4.39 -from time import gmtime,  strftime
    4.40 -from helper_functions import sql_functions
    4.41 -
    4.42 -def clear_temp_folder():
    4.43 -    """
    4.44 -    Clean the temporary folder used for query data, from files that are older than one day
    4.45 -    
    4.46 -    """
    4.47 -    
    4.48 -    try:    
    4.49 -        path = (cur_static_dir + 'temp/')
    4.50 -        listing = os.listdir(path)
    4.51 -        curtime = time.time()
    4.52 -        for infile in listing:
    4.53 -            temp_file = path + infile
    4.54 -            ftime = os.path.getmtime(temp_file)
    4.55 -            difftime = curtime - ftime
    4.56 -            if difftime > 86400:
    4.57 -                if not os.path.isdir(temp_file):
    4.58 -                    os.unlink(temp_file)
    4.59 -    except Exception as e:
    4.60 -        logging.exception('Exception caught:')
    4.61 -       
    4.62 -       
    4.63 -       
    4.64 -
    4.65 -
    4.66 -def change_destruction_time(url,new_time):
    4.67 -    """ 
    4.68 -    Change the destruction time for a TAP job
    4.69 -
    4.70 -    ATTENTION: Currently unused in this version
    4.71 -    
    4.72 -    @param url: The URL for the TAP job to destroy
    4.73 -    @param new_time : The current time to set for the destruction of the TAP job
    4.74 -    """
    4.75 -    
    4.76 -    f = ''
    4.77 -    try:
    4.78 -        params = urllib.urlencode({'DESTRUCTION' : new_time}) 
    4.79 -        url = url+'/destruction'
    4.80 -        req = urllib2.Request(url, params)
    4.81 -        f = urllib2.urlopen(req)
    4.82 -        f.close()
    4.83 -    except Exception:
    4.84 -        if f!='':
    4.85 -            f.close()
    4.86 -        logging.exception('Exception caught:')
    4.87 -
    4.88 - 
    4.89 - 
    4.90 -
    4.91 -
    4.92 -
    4.93 -def generate_form_data(registry):
    4.94 -    """
    4.95 -    Fetch array of available TAP endpoints from the input registry. Uses XQuery, and returns a tuple of web.form.Checkbox objects, to be added to the tap endpoint selection form
    4.96 -    
    4.97 -    @param registry: A string containin the Registry URL
    4.98 -    @return: An tuple containing the registry endpoints as web.checkbox elements
    4.99 -    """
   4.100 -
   4.101 -    url_query = """
   4.102 -    //RootResource[@status='active' and capability[@standardID='ivo://ivoa.net/std/TAP']]/capability[@standardID='ivo://ivoa.net/std/TAP']/interface/accessURL
   4.103 -    """
   4.104 -
   4.105 -
   4.106 -    title_query = """
   4.107 -    //RootResource[@status='active' and capability[@standardID='ivo://ivoa.net/std/TAP']]/title
   4.108 -    """ 
   4.109 -
   4.110 -  
   4.111 -    form_data=""
   4.112 -    title_results = []
   4.113 -    tap_url_results = []
   4.114 -    f=''
   4.115 -    try:
   4.116 -        # get title elements
   4.117 -        params = urllib.urlencode({'performquery' : 'true', 'ResourceXQuery' : title_query}) 
   4.118 -        req = urllib2.Request(registry, params)
   4.119 -        f = urllib2.urlopen(req)
   4.120 -        result_string = f.read()
   4.121 -       
   4.122 -        RE_XML_ILLEGAL = u'([\u0000-\u0008\u000b-\u000c\u000e-\u001f\ufffe-\uffff])' + \
   4.123 -                 u'|' + \
   4.124 -                 u'([%s-%s][^%s-%s])|([^%s-%s][%s-%s])|([%s-%s]$)|(^[%s-%s])' % \
   4.125 -                  (unichr(0xd800),unichr(0xdbff),unichr(0xdc00),unichr(0xdfff),
   4.126 -                   unichr(0xd800),unichr(0xdbff),unichr(0xdc00),unichr(0xdfff),
   4.127 -                   unichr(0xd800),unichr(0xdbff),unichr(0xdc00),unichr(0xdfff))
   4.128 -        result_string = re.sub(RE_XML_ILLEGAL, "?", result_string)
   4.129 -        
   4.130 -        result_string = result_string.decode('latin-1')
   4.131 -        dom = xml.dom.minidom.parseString(result_string.encode('utf-8'))
   4.132 -        title_results=dom.getElementsByTagName("title")
   4.133 -
   4.134 -        # get TAP urls
   4.135 -        params2 = urllib.urlencode({'performquery' : 'true', 'ResourceXQuery' : url_query}) 
   4.136 -        req2 = urllib2.Request(registry, params2)
   4.137 -        f2 = urllib2.urlopen(req2)
   4.138 -        result_string2 = f2.read()
   4.139 -        dom2 = xml.dom.minidom.parseString(result_string2)
   4.140 -        tap_url_results=dom2.getElementsByTagName("accessURL")
   4.141 -      
   4.142 -        counter = 0
   4.143 -        for i in tap_url_results:
   4.144 -            form_data+='<li><span>' + xml_parser.getText(title_results[counter]) + '</span> as <input type="text" name="catalogue"  value="'  + xml_parser.getText(title_results[counter]) +  '" /><input type="hidden" name="ident" value="' + xml_parser.getText(i) + '"/>' +  remove_catalogue_html + '</li>'
   4.145 -            counter = counter +1
   4.146 -            
   4.147 -        f.close()
   4.148 -        f2.close()
   4.149 -    
   4.150 -    except Exception as e:
   4.151 -        if f!='':
   4.152 -            f.close()   
   4.153 -        logging.exception('Exception caught:')      
   4.154 -        return ""
   4.155 -    if form_data=="":
   4.156 -        return ""
   4.157 -    else:
   4.158 -        return form_data
   4.159 -
   4.160 -
   4.161 -
   4.162 -
   4.163 -
   4.164 -
   4.165 -def generate_endpoint_from_list(endpnt_list):
   4.166 -    """
   4.167 -    Takes an list of TAP endpoint URLs separated by a double space, and generates a merged TAP service from the OGSDA-DAI TAP factory(global tap_factory variable)
   4.168 -           
   4.169 -    @param endpnt_lits: A list of available TAP endpoints as strings
   4.170 -    @param prefixes: A list of prefixes as a json object, from which we extract URL & name and submit to the Registry
   4.171 -    @return: A string with the new URL to be used by the client
   4.172 -    """
   4.173 -    
   4.174 -    data = ""
   4.175 -    result= ""
   4.176 -    urls = endpnt_list
   4.177 -    response = ''
   4.178 -    counter = 0
   4.179 -    query_schema = ''
   4.180 -    importname=""
   4.181 -    
   4.182 -    from datetime import datetime
   4.183 -    t = datetime.now()
   4.184 -   
   4.185 -    try:
   4.186 -       
   4.187 -        ### Create workspace
   4.188 -        workspace_name = 'workspace-' + t.strftime("%y%m%d_%H%M%S") 
   4.189 -        data = urllib.urlencode({firethorn_config.resource_create_name_params['http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json'] : workspace_name})
   4.190 -        req = urllib2.Request( firethorn_config.workspace_creator, data, headers={"Accept" : "application/json", "firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : session.get("community_input","public (unknown)")})
   4.191 -        response = urllib2.urlopen(req)
   4.192 -        workspace = json.loads(response.read())["ident"]
   4.193 -        response.close()
   4.194 -        
   4.195 -        ### Create Query Schema 
   4.196 -        data = urllib.urlencode({firethorn_config.resource_create_name_params['http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-schema-1.0.json'] : "query_schema"})
   4.197 -        req = urllib2.Request( workspace +  firethorn_config.schema_create_uri, data, headers={"Accept" : "application/json", "firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : session.get("community_input","public (unknown)")})
   4.198 -        response = urllib2.urlopen(req) 
   4.199 -        query_schema = json.loads(response.read())["ident"]
   4.200 -        response.close()
   4.201 -        
   4.202 -      
   4.203 -        for i in urls:
   4.204 -            importname = ""
   4.205 -            name = i['name']
   4.206 -            alias = i['alias']
   4.207 -            ident = i['ident']
   4.208 -   
   4.209 -        
   4.210 -            if alias!="":
   4.211 -                importname = alias
   4.212 -            else :
   4.213 -                importname = name
   4.214 -        
   4.215 -            if importname!="":
   4.216 -                data = urllib.urlencode({'urn:adql.copy.depth' : "THIN", firethorn_config.workspace_import_schema_base : ident, firethorn_config.workspace_import_schema_name : importname})
   4.217 -            else :
   4.218 -                data = urllib.urlencode({'urn:adql.copy.depth' : "THIN", firethorn_config.workspace_import_schema_base : ident})
   4.219 -            
   4.220 -          
   4.221 -            req = urllib2.Request( workspace + firethorn_config.workspace_import_uri, data, headers={"Accept" : "application/json", "firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : session.get("community_input","public (unknown)")}) 
   4.222 -            response = urllib2.urlopen(req)
   4.223 -            response.close()
   4.224 -            
   4.225 -            
   4.226 -    except Exception:
   4.227 -        if response!='':
   4.228 -            response.close()
   4.229 -            result = "" 
   4.230 -        logging.exception("IOError")  
   4.231 -    
   4.232 -  
   4.233 -      
   4.234 -    return query_schema
   4.235 -
   4.236 -
   4.237 -
   4.238 -
   4.239 -#######################################
   4.240 -# Note: Not used in Firethorn version #
   4.241 -#######################################
   4.242 -def fetch_metadata(tap_endpoint):
   4.243 -    """
   4.244 -    Takes a TAP endpoint URL, and generates the table metadata. This is done by reading and parsing the XML table information using the xml_dom.minidom library and the xml_parser module
   4.245 -    
   4.246 -    @param tap_endpoint: A string with the TAP endpoint
   4.247 -    @return: HTML content with the metadata for the given URL
   4.248 -    """
   4.249 -    
   4.250 -    return_str = ''
   4.251 -    f = ''
   4.252 -    try:
   4.253 -        f = urllib.urlopen(tap_endpoint)
   4.254 -        dom_str = f.read()
   4.255 -        dom = xml.dom.minidom.parseString(dom_str)
   4.256 -        return_str = xml_parser.handleXML(dom)
   4.257 -        f.close()
   4.258 -    except Exception as e:
   4.259 -        if f!='':
   4.260 -            f.close()
   4.261 -        return_str =  ""   
   4.262 -        logging.exception(e)   
   4.263 -    return return_str
   4.264 -
   4.265 -
   4.266 -     
   4.267 -def get_children_metadata(ident, _type):
   4.268 - 
   4.269 -    import type_helpers
   4.270 -    from datetime import datetime
   4.271 -    
   4.272 -    error_processing = "<div class='error'>Unable to fetch the table metadata</div>"
   4.273 -    return_val = ''
   4.274 -    f= ""
   4.275 -    ident = string_functions.decode(ident)
   4.276 -    _type = string_functions.decode(_type)
   4.277 -    
   4.278 -    try:
   4.279 -        if ident!="" and _type!="":
   4.280 -            if not type_helpers.isColumn(_type):
   4.281 -                ident = ident + firethorn_config.resource_uris[_type]
   4.282 -            request = urllib2.Request(ident, headers={"Accept" : "application/json"})
   4.283 -            f = urllib2.urlopen(request)
   4.284 -            json_data = json.loads(f.read())
   4.285 -           
   4.286 -            if json_data == [] or json_data== None:
   4.287 -                return_val = "No data found"
   4.288 -            else:
   4.289 -                if type_helpers.isRootType(_type):
   4.290 -                    return_val += '<p class="accordion_table_title">Catalogues:</p>'
   4.291 -                if type_helpers.isSchema(_type):
   4.292 -                    return_val += '<p class="accordion_table_title">Tables:</p>'
   4.293 -                
   4.294 -                if type_helpers.isTable(_type):
   4.295 -                                        
   4.296 -                    return_val += '<p class="accordion_table_title">Columns:</p>'
   4.297 -                
   4.298 -                if type_helpers.isColumn(_type):
   4.299 -
   4.300 -                    return_val += "<div>"
   4.301 -                    return_val += "Data Type: " + str(json_data["meta"]["adql"]["type"]) +"</br>"
   4.302 -                    return_val += "Size: " +  str(json_data["meta"]["adql"]["arraySize"])  +"</br>"
   4.303 -                    return_val += "UCD  Type: "  + str(json_data["meta"]["adql"]["ucd"]["type"]) + " - UCD Value: " + str(json_data["meta"]["adql"]["ucd"]["value"]) + "</br>"
   4.304 -                    return_val += "Unit: "  + str(json_data["meta"]["adql"]["units"]) +  "</br>"
   4.305 -                    return_val += "Utype: "  + str(json_data["meta"]["adql"]["utype"]) +  "</br>"
   4.306 -                    return_val += "</br></div>"
   4.307 -                    return_val += "</br>"
   4.308 -                    
   4.309 -                else:
   4.310 -                    
   4.311 -                    for entry in json_data: 
   4.312 -                        
   4.313 -                       
   4.314 -                        if type_helpers.isRootType(_type):
   4.315 -                            converted_dict = dict([(str(k), v) for k, v in entry.items()])
   4.316 -                            
   4.317 -                          
   4.318 -                            if converted_dict["fullname"]!='query_schema':
   4.319 -                                return_val+='<div id="accordion_catalogue">'
   4.320 -                                jdbcname_url  = converted_dict["root"]
   4.321 -                                request = urllib2.Request(jdbcname_url, headers={"Accept" : "application/json"})
   4.322 -                                f = urllib2.urlopen(request)
   4.323 -                                jdbcname_data = json.loads(f.read())
   4.324 -                                jdbcname = jdbcname_data["fullname"]
   4.325 -                                if jdbcname.endswith('.dbo'):
   4.326 -                                    jdbcname = jdbcname[:-4]
   4.327 -                                
   4.328 -                                return_val += '<input type="hidden" name="resource_type" value="' + converted_dict["type"]+ '" /> <input type="hidden" name="resource_ident" value="' + string_functions.encode(converted_dict["ident"]) +'" />'   +  "<div  class='heading'>" + converted_dict["fullname"] + '<img style="float:right;margin-top:5px" src="' + survey_prefix + '/static/static_vo_tool/asc.gif"/></div>'
   4.329 -                                return_val += "<div id='item_description' style='display:none'> Description: "  + html_functions.escape(str(converted_dict["text"])) + ' <br /> Full Database name:' + jdbcname + ' </div>'
   4.330 -                                return_val+='</div>'
   4.331 -                        elif type_helpers.isSchema(_type):
   4.332 -                            return_val+='<div id="accordion_schema">'
   4.333 -                            converted_dict = dict([(str(k), v) for k, v in entry.items()])
   4.334 -                            return_val += '<input type="hidden" name="resource_type" value="' + converted_dict["type"]+ '" /> <input type="hidden" name="resource_ident" value="' + string_functions.encode(converted_dict["ident"]) +'" />'   + "<div  class='heading'>" + converted_dict["fullname"] + '  <img style="float:right;margin-top:5px" src="' + survey_prefix + '/static/static_vo_tool/asc.gif"/></div>'
   4.335 -                            return_val += "<div id='item_description' style='display:none'> Description: "  + html_functions.escape(str(converted_dict["text"])) +'</div>'
   4.336 -                            return_val+='</div>'
   4.337 -                        elif type_helpers.isTable(_type):
   4.338 -                            return_val+='<div id="accordion_table">'
   4.339 -                         
   4.340 -                            converted_dict = dict([(str(k), v) for k, v in entry.items()])
   4.341 -                            return_val += '<input type="hidden" name="resource_type" value="' + converted_dict["type"]+ '" /> <input type="hidden" name="resource_ident" value="' + string_functions.encode(converted_dict["ident"]) +'" />'   + "<div  class='heading'>" + converted_dict["fullname"] + '  <img style="float:right;margin-top:5px" src="' + survey_prefix + '/static/static_vo_tool/asc.gif"/></div>'
   4.342 -                            return_val += "<div id='item_description'style='display:none'> Description: "  + html_functions.escape(str(converted_dict["text"])) + '<br /> Fullname: '  + str(converted_dict["fullname"]) +'</div>'
   4.343 -                            return_val+='</div>'
   4.344 -                       
   4.345 -                    
   4.346 -    except Exception:
   4.347 -        if f!="":
   4.348 -            f.close()    
   4.349 -        logging.exception("Error fetching metadata")
   4.350 -        return error_processing
   4.351 -    
   4.352 -    finally:
   4.353 -        if f!="":
   4.354 -            f.close()  
   4.355 -   
   4.356 -    return return_val
   4.357 -
   4.358 -def get_async_results(url, URI):
   4.359 -    """
   4.360 -    Open the given url and URI and read/return the result 
   4.361 -
   4.362 -    @param url: A URL string to open
   4.363 -    @param URI: A URI string to attach to the URL request
   4.364 -    @return: The result of the HTTP request sent to the URI of the URL
   4.365 -    """
   4.366 -    
   4.367 -    res = ''
   4.368 -    f = ''
   4.369 -    try:
   4.370 -        req = urllib2.Request(url+URI)
   4.371 -        f = urllib2.urlopen(req)
   4.372 -        res =  f.read()
   4.373 -        f.close()
   4.374 -    except Exception as e:
   4.375 -        if f!='':
   4.376 -            f.close()
   4.377 -        logging.exception('Exception caught:')
   4.378 -              
   4.379 -    return res
   4.380 -
   4.381 -
   4.382 -
   4.383 -def start_query_loop(url):
   4.384 -    """
   4.385 -    
   4.386 -    @param url: A URL string to be used
   4.387 -    @return: Results of query
   4.388 -    """
   4.389 -    
   4.390 -    def get_status(url):
   4.391 -        request2 = urllib2.Request(url, headers={"Accept" : "application/json", "firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : session.get("community_input","public (unknown)")})
   4.392 -        f_read = urllib2.urlopen(request2)
   4.393 -        query_json = f_read.read()
   4.394 -        return query_json
   4.395 -    
   4.396 -    max_size_exceeded = False 
   4.397 -   
   4.398 -    f_read = ""
   4.399 -    return_vot = ''
   4.400 -    delay = INITIAL_DELAY
   4.401 -    start_time = time.time()
   4.402 -    elapsed_time = 0
   4.403 -    query_json = {'syntax' : {'friendly' : 'A problem occurred while running your query', 'status' : 'Error' }}
   4.404 -    
   4.405 -    try:
   4.406 -        logging.exception("Started Firethorn job :::" +  strftime("%Y-%m-%d %H:%M:%S", gmtime()))
   4.407 -
   4.408 -        data = urllib.urlencode({ firethorn_config.query_status_update : "RUNNING"})
   4.409 -        
   4.410 -        #data = urllib.urlencode({ firethorn_config.query_status_update : "RUNNING", 'adql.query.update.delay.every' : '10000', 'adql.query.update.delay.first':'10000', 'adql.query.update.delay.last':'10000'})
   4.411 -        request = urllib2.Request(url, data, headers={"Accept" : "application/json", "firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : session.get("community_input","public (unknown)")})
   4.412 -        f_update = urllib2.urlopen(request)
   4.413 -        query_json =  json.loads(f_update.read())
   4.414 -        query_status = query_json["status"]
   4.415 -
   4.416 -        while query_status=="PENDING" or query_status=="RUNNING" and elapsed_time<MAX_ELAPSED_TIME:
   4.417 -            query_json = json.loads(get_status(url))
   4.418 -            query_status= query_json["status"] 
   4.419 -            time.sleep(delay)
   4.420 -            if elapsed_time>MIN_ELAPSED_TIME_BEFORE_REDUCE and delay<MAX_DELAY:
   4.421 -                delay = delay + delay
   4.422 -            elapsed_time = int(time.time() - start_time)
   4.423 - 
   4.424 -      
   4.425 -        if query_status=="ERROR" or query_status=="FAILED":
   4.426 -            return {'Code' :-1,  'Content' : 'Query error: A problem occurred while running your query' }
   4.427 -        elif query_status=="CANCELLED":
   4.428 -            return {'Code' :1,  'Content' : 'Query error: Query has been canceled' }
   4.429 -        elif query_status=="EDITING":
   4.430 -            logging.exception( "Editing status in start_query_loop:" )
   4.431 -            return {'Code' :-1,  'Content' : 'Query error: ' + query_json["syntax"]["status"] + ' - ' + query_json["syntax"]["friendly"] }
   4.432 -        elif query_status=="COMPLETED":
   4.433 -            return {'Code' :1,  'Content' : query_json["results"]["datatable"] }
   4.434 -        elif elapsed_time>=MAX_ELAPSED_TIME:
   4.435 -            return {'Code' :-1,  'Content' : 'Query error: Max run time (' + str(MAX_ELAPSED_TIME) + ' seconds) exceeded' }
   4.436 -        else:
   4.437 -            return {'Code' :-1,  'Content' : 'Query error: A problem occurred while running your query' }
   4.438 -        
   4.439 -    
   4.440 -    except Exception as e:
   4.441 -        logging.exception('Exception caught in start query loop:')  
   4.442 -        logging.exception(e)     
   4.443 -        return []
   4.444 -
   4.445 -    if f_read != "":
   4.446 -        f_read.close() 
   4.447 -    logging.exception("return_vot in start_query_loop: " + return_vot)
   4.448 -    return return_vot    
   4.449 -
   4.450 -
   4.451 -
   4.452 -def start_async_loop(url):
   4.453 -    """
   4.454 -    Takes a TAP url and starts a loop that checks the phase URI and returns the results when completed. The loop is repeated every [delay=3] seconds
   4.455 -
   4.456 -    @param url: A URL string to be used
   4.457 -    @return: A Votable with the results of a TAP job, or '' if error
   4.458 -    """
   4.459 -    
   4.460 -    max_size_exceeded = False 
   4.461 -    f = ""
   4.462 -    return_vot = ''
   4.463 -    try:
   4.464 -        while True:
   4.465 -            res = get_async_results(url,'/phase')
   4.466 -            if res=='COMPLETED':       
   4.467 -                req = urllib2.Request(url + '/results/result')
   4.468 -                f = urllib2.urlopen(req)
   4.469 -                f.read(MAX_FILE_SIZE) #100Mb = 104,857,600
   4.470 -                if len(f.read())>0:
   4.471 -                    max_size_exceeded = True
   4.472 -                if not max_size_exceeded:
   4.473 -                    return_vot = atpy.Table(url + '/results/result', type='vo')
   4.474 -           
   4.475 -                else:
   4.476 -                    return_vot = "MAX_ERROR" 
   4.477 -                break
   4.478 -            elif res=='ERROR' or res== '':
   4.479 -                return return_vot
   4.480 -            time.sleep(delay)
   4.481 -    except Exception as e:
   4.482 -        logging.exception('Exception caught:')
   4.483 -        
   4.484 -        if f != "":
   4.485 -            f.close()
   4.486 -        return_vot = ''
   4.487 -    return return_vot    
   4.488 -
   4.489 -        
   4.490 -
   4.491 -
   4.492 -
   4.493 -
   4.494 -def print_save_HTML_table_info(pathname):
   4.495 -    """
   4.496 -    Generates HTML content for HTML table save button
   4.497 -        
   4.498 -    @param tbl: A table with the results of a query as a JSON, escaped array
   4.499 -    @return: The HTML content with a save as button
   4.500 -    """
   4.501 -        
   4.502 -
   4.503 -    result = '<div id="save_as"><form name="save_as_html" action="' + survey_sub_path + 'save_as_html" method="post"> <input type="hidden" name="pathname" value ="' 
   4.504 -    result += pathname + '"/><input type="submit" class ="save_button" value="Save as HTML" /></form></div>'
   4.505 -    return result
   4.506 -    
   4.507 -    
   4.508 -
   4.509 -
   4.510 -
   4.511 -
   4.512 -def print_save_VOTable_info(pathname):    
   4.513 -    """
   4.514 -    Generates HTML content for VOTable save button    
   4.515 -
   4.516 -    @param vot: A table with the results of a query as a JSON, escaped array
   4.517 -    @return: The HTML content with a save as button
   4.518 -    """
   4.519 -
   4.520 -    result = '<div id="save_as"><form name="save_as_vot" action="' + survey_sub_path + 'save_as_vot" method="post"> <input type="hidden" name="pathname" value ="' 
   4.521 -    result += pathname + '"/><input type="submit" class ="save_button" value="Save as Votable" /></form></div>'
   4.522 -    return result
   4.523 -
   4.524 -
   4.525 -
   4.526 -
   4.527 -
   4.528 -def print_save_Fits_info(pathname):    
   4.529 -    """
   4.530 -    Generates HTML content for VOTable save button    
   4.531 -
   4.532 -    @param vot: A table with the results of a query as a JSON, escaped array
   4.533 -    @return: The HTML content with a save as button
   4.534 -    """
   4.535 -    
   4.536 -    result = '<div id="save_as"><form name="save_as_fits" action="' + survey_sub_path + 'save_as_fits" method="post"> <input type="hidden" name="pathname" value ="' 
   4.537 -    result += pathname + '"/><input type="submit" class ="save_button" value="Save as Fits" /></form></div>'
   4.538 -    return result
   4.539 -
   4.540 -
   4.541 -def execute_sql_query(query, database):
   4.542 -    """
   4.543 -    Execute an SQL query
   4.544 -    @param query: The SQL Query
   4.545 -
   4.546 -    """
   4.547 -    votable = ''
   4.548 -    jobId= 'None'
   4.549 -    file_path=''
   4.550 -    rows=[]
   4.551 -    now = datetime.datetime.now()
   4.552 -    adql_table=''
   4.553 -    query_loop_results=''
   4.554 -    cols = []
   4.555 -    rows = []
   4.556 -    datatable=[]
   4.557 -    dthandler = lambda obj: (
   4.558 -        obj.isoformat()
   4.559 -        if isinstance(obj, datetime.datetime) or isinstance(obj, datetime.date) else None)
   4.560 -
   4.561 -    try:
   4.562 -
   4.563 -        datatable = sql_functions.execute_query_get_cols_rows(query, database)
   4.564 -       
   4.565 -
   4.566 -        if  datatable!=None :
   4.567 -         
   4.568 -            clear_temp_folder()
   4.569 -            file_handle = File_handler()
   4.570 -            file_handle.create_temp_file()
   4.571 -            file_handle.write_to_temp_file(json.dumps(datatable, default=dthandler))
   4.572 -            file_handle.close_handle()    
   4.573 -            file_path = file_handle.pathname
   4.574 -    except pyodbc.ProgrammingError, err:
   4.575 -        error_message = repr(err)
   4.576 -        error_message = error_message.split("[FreeTDS]",1)
   4.577 -        if len(error_message)>0:
   4.578 -            datatable = "Query Error: " + error_message [1][:-2] 
   4.579 -        else :
   4.580 -            datatable =  "Query Error: There was a server error while processing your request"
   4.581 -        return (datatable,jobId,file_path,adql_table, query_loop_results)     
   4.582 -    except Exception as e:
   4.583 -        datatable = "Query Error: There was a server error while processing your request"
   4.584 -        logging.exception(e) 
   4.585 -        return (datatable,jobId,file_path,adql_table, query_loop_results)
   4.586 -    logging.exception("Completed run_query :::" +  strftime("%Y-%m-%d %H:%M:%S", gmtime()))
   4.587 -    
   4.588 -    return (json.dumps(datatable, default=dthandler),jobId,file_path,adql_table, query_loop_results)
   4.589 -    
   4.590 -    
   4.591 -    
   4.592 -
   4.593 -def execute_async_query(url,mode_local,q,_format="votable"):
   4.594 -    """
   4.595 -    Execute an ADQL query (q) against a TAP service (url + mode:sync|async)       
   4.596 -    Starts by submitting a request for an async query, then uses the received job URL to call start_async_loop, to receive the final query results 
   4.597 -
   4.598 -    @param url: A string containing the TAP URL
   4.599 -    @param mode: sync or async to determine TAP mode of execution
   4.600 -    @param q: The ADQL Query to execute as string
   4.601 -    
   4.602 -    @return: Return a votable with the results, the TAP job ID and a temporary file path with the results stored on the server
   4.603 -    """    
   4.604 -
   4.605 -    votable = ''
   4.606 -    jobId= 'None'
   4.607 -    file_path=''
   4.608 -    rows=[]
   4.609 -    now = datetime.datetime.now()
   4.610 -    adql_table=''
   4.611 -    query_loop_results=''
   4.612 -    cols = []
   4.613 -    rows = []
   4.614 -    
   4.615 -    try:
   4.616 -
   4.617 -        # Return results as a votable object
   4.618 -        logging.exception("Started run_query :::" +  strftime("%Y-%m-%d %H:%M:%S", gmtime()))
   4.619 -        votable, adql_table, query_loop_results = run_query(q,"",query_space=url)
   4.620 -        if votable!='' and votable !="MAX_ERROR" and votable !="ERROR" and votable!=None :
   4.621 -            if ((type(votable) is str) or (type(votable) is unicode)):
   4.622 -                if votable.startswith("Query"):
   4.623 -                    return (votable,jobId,file_path,adql_table, query_loop_results)
   4.624 -            clear_temp_folder()
   4.625 -           
   4.626 -            #cols = list(votable.columns)
   4.627 -            '''
   4.628 -            lst = []
   4.629 -            
   4.630 -            #rows = votable.data.tolist()
   4.631 -            #row_length = len(rows)
   4.632 -            
   4.633 -            
   4.634 -            for x in rows:
   4.635 -                second_counter = 0
   4.636 -                obj_list = {}
   4.637 -                for y in x:
   4.638 -                  
   4.639 -                    obj_list[cols[second_counter]] = y 
   4.640 -                    second_counter = second_counter + 1
   4.641 -                lst.append(obj_list)
   4.642 -            lst = [cols,lst] 
   4.643 -            '''
   4.644 -            
   4.645 -            file_handle = File_handler()
   4.646 -            file_handle.create_temp_file()
   4.647 -            file_handle.write_to_temp_file(votable)
   4.648 -            file_handle.close_handle()    
   4.649 -            file_path = file_handle.pathname
   4.650 -            
   4.651 -    except Exception as e:
   4.652 -        logging.exception('Exception caught:')
   4.653 -        logging.exception(e) 
   4.654 -
   4.655 -    logging.exception("Completed run_query :::" +  strftime("%Y-%m-%d %H:%M:%S", gmtime()))
   4.656 -
   4.657 -    return (votable,jobId,file_path,adql_table, query_loop_results)
   4.658 -
   4.659 -
   4.660 -
   4.661 -
   4.662 -def run_query(query=None, query_name="", query_space="", **kwargs):
   4.663 -    """
   4.664 -    Run a query on a resource
   4.665 -    """
   4.666 -    f=''
   4.667 -  
   4.668 -    query =  query
   4.669 -    query_space = string_functions.decode(query_space)
   4.670 -    result = ""
   4.671 -    max_size_exceeded = False
   4.672 -    result_adql_table = ""
   4.673 -    query_identity = ""
   4.674 -    
   4.675 -    try :
   4.676 -        from datetime import datetime
   4.677 -        t = datetime.now()
   4.678 -        if query_name=="":
   4.679 -            query_name = 'query-' + t.strftime("%y%m%d_%H%M%S")
   4.680 -     
   4.681 -        #data = urllib.urlencode({ firethorn_config.query_name_param : query_name,  firethorn_config.query_param : query,'adql.query.update.delay.every' : '10000', 'adql.query.update.delay.first':'10000', 'adql.query.update.delay.last':'10000'})
   4.682 -        
   4.683 -        urlenc = { firethorn_config.query_name_param : query_name,  firethorn_config.query_param : query}
   4.684 -        data = urllib.urlencode(urlenc)
   4.685 -        request = urllib2.Request(query_space + firethorn_config.query_create_uri, data, headers={"Accept" : "application/json", "firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : session.get("community_input","public (unknown)")})
   4.686 -
   4.687 -        f = urllib2.urlopen(request)
   4.688 -        query_create_result = json.loads(f.read())
   4.689 -        query_identity = query_create_result["ident"]
   4.690 -        query_loop_results = start_query_loop(query_identity)
   4.691 -        results_adql_url = query_create_result["results"]["adql"]
   4.692 -
   4.693 -        
   4.694 -        if query_loop_results.get("Code", "") !="":
   4.695 -            if query_loop_results.get("Code", "") ==-1:
   4.696 -                result = query_loop_results.get("Content", "Error")
   4.697 -                
   4.698 -                return (result,result_adql_table,query_identity)
   4.699 -        
   4.700 -        if results_adql_url!=None:
   4.701 -               
   4.702 -            try :
   4.703 -                req_rez_table = urllib2.Request( results_adql_url, headers={"Accept" : "application/json", "firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : session.get("community_input","public (unknown)")})
   4.704 -                response_rez_table = urllib2.urlopen(req_rez_table) 
   4.705 -                response_rez_table_json = response_rez_table.read()
   4.706 -                result_adql_table = json.loads(response_rez_table_json)["fullname"]
   4.707 -                response_rez_table.close()
   4.708 -            except Exception as e:
   4.709 -                logging.exception(e)
   4.710 -        
   4.711 -         
   4.712 -        if query_loop_results.get("Code", "") !="":
   4.713 -            if query_loop_results.get("Code", "") ==-1:
   4.714 -                result = query_loop_results.get("Content", "Error")
   4.715 -            elif query_loop_results.get("Code", "") ==1:
   4.716 -                req = urllib2.Request(query_loop_results.get("Content", ""), headers={"firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : session.get("community_input","public (unknown)")})
   4.717 -                f = urllib2.urlopen(req)
   4.718 -                datatable = f.read() 
   4.719 -             
   4.720 -                if len(f.read())>0:
   4.721 -                    max_size_exceeded = True
   4.722 -                if not max_size_exceeded:
   4.723 -                    result = datatable
   4.724 -                else:
   4.725 -                    result = ("MAX ERROR",result_adql_table,query_identity)
   4.726 -
   4.727 -        else :
   4.728 -      
   4.729 -            return ("ERROR",result_adql_table,query_identity)
   4.730 -            
   4.731 -    except Exception as e:
   4.732 -        logging.exception('Exception caught in run query:')
   4.733 -        logging.exception(e) 
   4.734 -        return ("ERROR",result_adql_table,query_identity)
   4.735 -    
   4.736 -    if f!='':
   4.737 -        f.close()
   4.738 -    return (result,result_adql_table,query_identity)
   4.739 -    
   4.740 -
   4.741 -def imagelist_generate_JSON_from_query((votable,jobId,pathname,adql_table,query_results_id),query="",tap_endpoint="",table_mode="static"):    
   4.742 -    """
   4.743 -    Generates HTML content for the results from an ADQL/TAP query    
   4.744 -    
   4.745 -    @param (votable,jobId,pathname): A tuple with the votable results, the jobID and the temporary file path
   4.746 -    @param query: The query that was executed
   4.747 -    @param tap_endpoint: The currently selected or generated TAP endpoint URL
   4.748 -    @return: The HTML content to be returned and asynchronously loaded on the users page after submitting a query
   4.749 -    """
   4.750 -    
   4.751 -    data_results = ""
   4.752 -    json_list = ""
   4.753 -    content = ""
   4.754 -
   4.755 -    if votable == "MAX_ERROR" or votable == "ERROR" or (getattr(votable, "columns", None)==None and (votable!='' and votable!= None)):
   4.756 -	data_results = ["ERROR"]	
   4.757 -	if votable == "MAX_ERROR":
   4.758 -	    data_results.append("Query exceeded maximum return size limit")
   4.759 -	elif votable == "ERROR":
   4.760 -  	    data_results.append("There was an error processing your request")
   4.761 -	else:	
   4.762 -	    data_results.append(votable)    
   4.763 -	if query_results_id!="" and query_results_id!=None:
   4.764 -	    query_id_content = '<div style="height:33px;text-align:left;margin-left:20px;font-size:13px;color:#C2BEAD">Query ID <a id="toggle_query_info">[+]</a><a id="toggle_query_info_help">[?]</a><div id="toggle_query_info_div" style="display:none">'+ query_results_id.split("/")[-1]  + '</div></div><br>' 
   4.765 -	else :
   4.766 -	    query_id_content = ""
   4.767 -	data_results.append(query_id_content)
   4.768 -	
   4.769 -        return json.dumps(data_results)
   4.770 -    elif votable!='' and votable!= None:
   4.771 -        if len(votable)>0:
   4.772 -            cols = list(votable.columns)
   4.773 -            row_list = votable.data.tolist() 
   4.774 -            if tap_endpoint!="" and query!="":
   4.775 -                content = '<form class="launch_viewer" style="float:left;z-index:100;position: relative;" action="' + survey_prefix + '/viewer" method="post" target="_blank"><input type="hidden" name="query" value="'+ string_functions.encode(query) +'"/><input type="hidden" name="cols" value="'+ html_functions.escape(json.dumps(cols)) +'"/><input type="hidden" name="filepath" id="temp_file" value="'+ pathname +'"/><input type="hidden" name="tap_endpoint" value="'+ tap_endpoint +'"/>'
   4.776 -                content += '<input type="hidden" name="adql_table" id="adql_table" value="'+ adql_table +'"/>'
   4.777 -                if table_mode == 'interactive':
   4.778 -                    data_results = html_functions.escape_list(row_list)
   4.779 -                    json_list = html_functions.escape(json.dumps(data_results))
   4.780 -                    content += '<input type="hidden" id="hidden_json_results" name="results" value="' + json_list  + '" />'
   4.781 -                content += '<div style="margin-bottom:5px;text-align:left;margin-left:20px;font-size:13px;color:#C2BEAD">Launch in Plotter<input type="submit" value="" /></form></div><br /><br />'
   4.782 -                content += '<div style="height:33px;clear:both;text-align:left;margin-left:20px;font-size:13px;color:#C2BEAD">Query ID <a id="toggle_query_info">[+]</a><a id="toggle_query_info_help">[?]</a><div id="toggle_query_info_div" style="display:none">'+ query_results_id.split("/")[-1]  + '</div></div><br>'
   4.783 -
   4.784 -            else:
   4.785 -                content = '<input type="hidden" name="filepath" id="temp_file" value="'+ pathname +'"/>'
   4.786 -            content += '<div id="save_as_information" style="display:none">' + print_save_VOTable_info(pathname) + print_save_HTML_table_info(pathname) + print_save_Fits_info(pathname) + '</div>'
   4.787 -            content += '<div style="text-align:left;margin-left:15px;float:left;z-index:100;position: relative;clear:both"><img id ="samp_connection" src="' + survey_prefix + '/static/images/red-button.png" height="17" width="17" /><button id="register">Connect to SAMP</button><button id="unregister" style="display: none;">Disconnect from SAMP</button><button id="loadvotable" style="display: none;">Broadcast results table</button></div>'
   4.788 -
   4.789 -            data_results = [cols]
   4.790 -            data_results.append([len(row_list)])
   4.791 -            data_results.append([content])
   4.792 -            return json.dumps(data_results)
   4.793 -        else:
   4.794 -            return json.dumps([])
   4.795 -    else:
   4.796 -
   4.797 -           
   4.798 -	data_results = ["ERROR"]
   4.799 -	if votable == "MAX_ERROR":
   4.800 -	    data_results.append("Query exceeded maximum return size limit")
   4.801 -	elif votable == "ERROR":
   4.802 -  	    data_results.append("There was an error processing your request")
   4.803 -	else:	
   4.804 -	    data_results.append(votable)    
   4.805 -
   4.806 -	if query_results_id!="" and query_results_id!=None:
   4.807 -	    query_id_content = '<div style="height:33px;text-align:left;margin-left:20px;font-size:13px;color:#C2BEAD">Query ID <a id="toggle_query_info">[+]</a><a id="toggle_query_info_help">[?]</a><div id="toggle_query_info_div" style="display:none"> '+ query_results_id.split("/")[-1]  + '</div></div><br>' 
   4.808 -	else :
   4.809 -	    query_id_content = ""
   4.810 -
   4.811 -	data_results.append(query_id_content)     
   4.812 -	return data_results
   4.813 -
   4.814 -    
   4.815 -
   4.816 -def generate_JSON_from_query((votable,jobId,pathname,adql_table,query_results_id),query="",tap_endpoint="",table_mode="static", mode="adql"):    
   4.817 -    """
   4.818 -    Generates HTML content for the results from an ADQL/TAP query    
   4.819 -    
   4.820 -    @param (votable,jobId,pathname): A tuple with the votable results, the jobID and the temporary file path
   4.821 -    @param query: The query that was executed
   4.822 -    @param tap_endpoint: The currently selected or generated TAP endpoint URL
   4.823 -    @return: The HTML content to be returned and asynchronously loaded on the users page after submitting a query
   4.824 -    """
   4.825 -    
   4.826 -    data_results = ""
   4.827 -    content = ""
   4.828 -    json_data = []
   4.829 -    row_length = "-1"
   4.830 -    
   4.831 -    if votable!='' and votable!= None and not (((type(votable) is str) or (type(votable) is unicode)) and votable.startswith("Query")):
   4.832 -        json_data = json.loads(votable)
   4.833 -    
   4.834 -        if len(json_data)<2:
   4.835 -            votable = "ERROR"
   4.836 -        else :
   4.837 -            cols = json_data[0]
   4.838 -            row_length = len(json_data[1])
   4.839 -    
   4.840 -    
   4.841 -    logging.exception("Starting generate_JSON_from_query :::" +  strftime("%Y-%m-%d %H:%M:%S", gmtime()))
   4.842 -
   4.843 -    if votable == "MAX_ERROR" or votable == "ERROR" or ( ((type(votable) is str) or (type(votable) is unicode)) and votable.startswith("Query")):
   4.844 -        data_results = ["ERROR"]	
   4.845 -        if votable == "MAX_ERROR":
   4.846 -            data_results.append("Query exceeded maximum return size limit")
   4.847 -        elif votable == "ERROR":
   4.848 -            data_results.append("There was an error processing your request")
   4.849 -        else:	
   4.850 -            data_results.append(votable)    
   4.851 -    
   4.852 -    
   4.853 -        if query_results_id!="" and query_results_id!=None:
   4.854 -            query_id_content = '<div style="height:33px;text-align:left;margin-left:20px;font-size:13px;color:#C2BEAD">Query ID <a id="toggle_query_info">[+]</a><a id="toggle_query_info_help">[?]</a><div id="toggle_query_info_div" style="display:none">'+ query_results_id.split("/")[-1]  + '</div></div><br>' 
   4.855 -        else :
   4.856 -            query_id_content = ""
   4.857 -        
   4.858 -        data_results.append(query_id_content)
   4.859 -        
   4.860 -        return json.dumps(data_results)
   4.861 -    
   4.862 -    elif votable!='' and votable!= None and not (((type(votable) is str) or (type(votable) is unicode)) and votable.startswith("Query")):
   4.863 -        if row_length>0:
   4.864 -            cols = list(cols)
   4.865 -            content=''
   4.866 -            if tap_endpoint!="" and query!="" and mode=='adql':
   4.867 -               
   4.868 -                content = '<form class="launch_viewer" style="float:left;z-index:100;position: relative;" action="' + survey_prefix + '/viewer" method="post" target="_blank"><input type="hidden" name="query" value="'+ string_functions.encode(query) +'"/><input type="hidden" name="cols" value="'+ html_functions.escape(json.dumps(cols)) +'"/><input type="hidden" name="filepath" id="temp_file" value="'+ pathname +'"/><input type="hidden" name="tap_endpoint" value="'+ tap_endpoint +'"/>'
   4.869 -                content += '<input type="hidden" name="adql_table" id="adql_table" value="'+ adql_table +'"/>'
   4.870 -                #if table_mode == 'interactive':
   4.871 -                #    data_results = html_functions.escape_list(row_list)
   4.872 -                #    json_list = html_functions.escape(json.dumps(data_results))
   4.873 -                #    content += '<input type="hidden" id="hidden_json_results" name="results" value="' + json_list  + '" />'
   4.874 -                content += '<div style="margin-bottom:5px;text-align:left;margin-left:20px;font-size:13px;color:#C2BEAD">Launch in Plotter<input type="submit" value="" /></form></div><br /><br />'
   4.875 -                content += '<div style="height:33px;clear:both;text-align:left;margin-left:20px;font-size:13px;color:#C2BEAD">Query ID <a id="toggle_query_info">[+]</a><a id="toggle_query_info_help">[?]</a><div id="toggle_query_info_div" style="display:none">'+ query_results_id.split("/")[-1]  + '</div></div><br>'
   4.876 -
   4.877 -            else:
   4.878 -                content = '<input type="hidden" name="filepath" id="temp_file" value="'+ pathname +'"/>'
   4.879 -            content += '<div id="save_as_information" style="display:none">' + print_save_VOTable_info(pathname) + print_save_HTML_table_info(pathname) + print_save_Fits_info(pathname) + '</div>'
   4.880 -            content += '<div style="text-align:left;margin-left:15px;float:left;z-index:100;position: relative;clear:both"><img id ="samp_connection" src="' + survey_prefix + '/static/images/red-button.png" height="17" width="17" /><button id="register">Connect to SAMP</button><button id="unregister" style="display: none;">Disconnect from SAMP</button><button id="loadvotable" style="display: none;">Broadcast results table</button></div>'
   4.881 -
   4.882 -            data_results = [cols]
   4.883 -            data_results.append([row_length])
   4.884 -            data_results.append([content])
   4.885 -            logging.exception("Completed generate_JSON_from_query :::" +  strftime("%Y-%m-%d %H:%M:%S", gmtime()))
   4.886 -            try:
   4.887 -                if debug_mode != True:
   4.888 -                    from web import ctx
   4.889 -                    now = datetime.datetime.now()
   4.890 -                    userip =  "None" if str(web.ctx.ip)==None  else  str(web.ctx.ip)
   4.891 -                    dbrelease = "Unknown" if query_results_id==None else query_results_id
   4.892 -        
   4.893 -                    MyOutputStream().log_queries(now.strftime('%Y-%m-%d %H:%M:%S.%f')[:-4], int((datetime.datetime.now() - now).total_seconds()), query, row_length,len(cols),html_functions.escape(session.get("username","unknown user")), html_functions.escape(dbrelease) , userip)
   4.894 -            except Exception:
   4.895 -                logging.exception('Exception caught:')
   4.896 -            return json.dumps(data_results)
   4.897 -        else:
   4.898 -            return json.dumps([])
   4.899 -    else:
   4.900 -
   4.901 -                 
   4.902 -        data_results = ["ERROR"]
   4.903 -        if votable == "MAX_ERROR":
   4.904 -            data_results.append("Query exceeded maximum return size limit")
   4.905 -        elif votable == "ERROR":
   4.906 -            data_results.append("There was an error processing your request")
   4.907 -        else:	
   4.908 -            data_results.append(votable)    
   4.909 -        
   4.910 -        if query_results_id!="" and query_results_id!=None:
   4.911 -            query_id_content = '<div style="height:33px;text-align:left;margin-left:20px;font-size:13px;color:#C2BEAD">Query ID <a id="toggle_query_info">[+]</a><a id="toggle_query_info_help">[?]</a><div id="toggle_query_info_div" style="display:none"> '+ query_results_id.split("/")[-1]  + '</div></div><br>' 
   4.912 -        else :
   4.913 -            query_id_content = ""
   4.914 -        
   4.915 -        data_results.append(query_id_content)    
   4.916 -        
   4.917 - 
   4.918 -        return data_results
   4.919 -
   4.920 -
   4.921 -
   4.922 -def execute_async_region_query(url,mode_local,q):
   4.923 -    """
   4.924 -    Execute an ADQL query (q) against a TAP service (url + mode:sync|async)       
   4.925 -    Starts by submitting a request for an async query, then uses the received job URL to call start_async_loop, to receive the final query results 
   4.926 -
   4.927 -    @param url: A string containing the TAP URL
   4.928 -    @param mode: sync or async to determine TAP mode of execution
   4.929 -    @param q: The ADQL Query to execute as string
   4.930 -    
   4.931 -    @return: Return a votable with the results, the TAP job ID and a temporary file path with the results stored on the server
   4.932 -    """    
   4.933 -
   4.934 -    params = urllib.urlencode({'REQUEST': request, 'LANG': lang, 'FORMAT': result_format, 'QUERY' : q}) 
   4.935 -    full_url = url+mode_local
   4.936 -
   4.937 -    votable = ''
   4.938 -    jobId= 'None'
   4.939 -    file_path=''
   4.940 -    try:
   4.941 -        
   4.942 -        #Submit job and get job id 
   4.943 -        req = urllib2.Request(full_url, params)
   4.944 -        opener = urllib2.build_opener()
   4.945 -  
   4.946 -        f = opener.open(req)
   4.947 -        jobId = f.url
   4.948 -  
   4.949 -        #Execute job and start loop requests for results
   4.950 -        req2 = urllib2.Request(jobId+'/phase',urllib.urlencode({'PHASE' : 'RUN'}))
   4.951 -        f2 = opener.open(req2) #@UnusedVariable
   4.952 -
   4.953 -        # Return results as a votable object
   4.954 -        votable = start_async_loop(jobId)
   4.955 -        
   4.956 -        if votable!='' and votable !="MAX_ERROR" and not (((type(votable) is str) or (type(votable) is unicode)) and votable.startswith("Query")):
   4.957 -            multiframeID_index = -1
   4.958 -            framesetID_index = -1
   4.959 -            dec_index = -1  
   4.960 -            ra_index = -1
   4.961 -            clear_temp_folder()
   4.962 -            cols = list(votable.columns)
   4.963 -            cols.insert(0,"getFSLink")
   4.964 -            col_data_to_list =  list(votable.columns)
   4.965 -            
   4.966 -            if 'ra' in col_data_to_list:
   4.967 -                ra_index = col_data_to_list.index('ra')
   4.968 -            else :
   4.969 -                ra_index = -1
   4.970 -                
   4.971 -            if 'dec' in col_data_to_list:
   4.972 -                dec_index = col_data_to_list.index('dec')
   4.973 -            else :
   4.974 -                dec_index = -1
   4.975 -                
   4.976 -            if  'frameSetID' in col_data_to_list:
   4.977 -                framesetID_index = col_data_to_list.index('frameSetID')
   4.978 -            else:
   4.979 -                framesetID_index = -1
   4.980 -                if 'multiframeID' in col_data_to_list:
   4.981 -                    multiframeID_index = col_data_to_list.index('multiframeID')
   4.982 -                else:
   4.983 -                    multiframeID_index = -1
   4.984 -
   4.985 -
   4.986 -            db = SURVEY_DB
   4.987 -           
   4.988 -            temp_row=[]
   4.989 -            for i in votable.data.tolist():
   4.990 -                if framesetID_index == -1 and multiframeID_index == -1:
   4.991 -                    getFSLink = ''
   4.992 -                elif framesetID_index == -1 and ra_index!=-1 and dec_index!=-1:
   4.993 -                    getFSLink = '<a target="getI" href="' + getImageURL +'?mode=show&ra=' + str(i[ra_index])  + '&dec=' + str(i[dec_index]) + '&mfid=' + str(i[ multiframeID_index]) + '&database=' + db + '&archive=' + archive_input +'">view</a>'
   4.994 -                    
   4.995 -                elif multiframeID_index == -1 and ra_index!=-1 and dec_index!=-1:
   4.996 -                    getFSLink = '<a target="getI" href="' + getImageURL +'?mode=show&ra=' + str(i[ra_index])  + '&dec=' + str(i[dec_index]) + '&fsid=' + str(i[framesetID_index]) + '&database=' + db + '&archive=' + archive_input +'">view</a>'   
   4.997 -                else :
   4.998 -                    getFSLink = '<a target="getI" href="' + getImageURL +'?mode=show&ra=' + str(i[ra_index])  + '&dec=' + str(i[dec_index]) + '&fsid=' + str(i[framesetID_index]) + '&database=' + db + '&archive=' + archive_input +'">view</a>'
   4.999 -                temp_row.append(getFSLink)
  4.1000 -           
  4.1001 -            votable.add_column(name="getFSLink",data=temp_row,position=0)
  4.1002 -            
  4.1003 -            
  4.1004 -            lst = []
  4.1005 -            for x in votable.data.tolist():
  4.1006 -                second_counter = 0
  4.1007 -                obj_list = {}
  4.1008 -                for y in x:
  4.1009 -                    obj_list[cols[second_counter]] = y 
  4.1010 -                    second_counter = second_counter + 1
  4.1011 -                lst.append(obj_list)
  4.1012 -            
  4.1013 -            #html_functions.escape_list(votable.data.tolist())
  4.1014 -            lst = [cols,lst] 
  4.1015 -            file_handle = File_handler()
  4.1016 -            file_handle.create_temp_file()
  4.1017 -            file_handle.write_to_temp_file(json.dumps(lst))
  4.1018 -            file_handle.close_handle()    
  4.1019 -            file_path = file_handle.pathname
  4.1020 -            
  4.1021 -    except Exception as e:
  4.1022 -        logging.exception('Exception caught:')
  4.1023 -
  4.1024 -    return (votable,jobId,file_path,"",jobId)
  4.1025 -
  4.1026 -
  4.1027 -
  4.1028 -def execute_async_imagelist_query(cols,rows):
  4.1029 -    """
  4.1030 -    Generate results for imagelist query
  4.1031 -    """    
  4.1032 -  
  4.1033 -  
  4.1034 -    votable = ''
  4.1035 -    file_path=''
  4.1036 -
  4.1037 -    try:
  4.1038 -        
  4.1039 -        if cols!=None and rows !=None:
  4.1040 -            clear_temp_folder()
  4.1041 - 
  4.1042 -            col_data_to_list =  cols
  4.1043 -            votable = atpy.Table()
  4.1044 -            cntr = 0
  4.1045 -            
  4.1046 -            for i in cols:
  4.1047 -                temp_row = []
  4.1048 -            
  4.1049 -                for y in rows:
  4.1050 -                    if type(y[cntr]) == long:
  4.1051 -                        temp_row.append(float(y[cntr]))
  4.1052 -                    elif type(y[cntr]) == int or type(y[cntr]) == float:
  4.1053 -                        temp_row.append(y[cntr])
  4.1054 -                    else:
  4.1055 -                        temp_row.append(str(y[cntr]))
  4.1056 -                   
  4.1057 -                votable.add_column(i,data=temp_row,position=cntr)
  4.1058 -                cntr = cntr + 1
  4.1059 -                
  4.1060 -            filename = col_data_to_list.index('filename')
  4.1061 -            catname = col_data_to_list.index('catname')
  4.1062 -            jpegBase = col_data_to_list.index('jpegBase')
  4.1063 -            ext = col_data_to_list.index('numDetectors')
  4.1064 -            mfid = col_data_to_list.index('multiframeID')
  4.1065 -            rID = 1
  4.1066 -
  4.1067 -            temp_row_view=[]
  4.1068 -            temp_row_img=[]
  4.1069 -            temp_row_cat=[]
  4.1070 -           
  4.1071 -            for i in votable.data.tolist():
  4.1072 -
  4.1073 -                view = '<a id="ImageList_facebox" target="display" href="' + survey_cgi_bin + 'display.cgi?file=' + str(i[filename]) + '&cat=' + str(i[catname]) + '&comp=' + str(i[jpegBase]) + '&noExt=' + str(i[ext]) + '&MFID=' + str(i[mfid]) + '&rID=' + str(rID) +'">view</a>'
  4.1074 -                img = '<a  class="ImageList_dllink" href="' + survey_cgi_bin + 'fits_download.cgi?file=' + str(i[filename]) +'&MFID=' + str(i[mfid]) + '&rID=' + str(rID) +'">FITS</a>'
  4.1075 -                cat = '<a  class="ImageList_dllink" href="' + survey_cgi_bin + 'fits_download.cgi?file=' + str(i[catname]) + '&amp;MFID=' + str(i[mfid]) + '&amp;rID=' + str(rID) + '">FITS</a>'
  4.1076 -                temp_row_view.append(view)
  4.1077 -                temp_row_img.append(img)
  4.1078 -                temp_row_cat.append(cat)
  4.1079 -            
  4.1080 -            votable.remove_columns(['jpegBase','filename','catname'])
  4.1081 -
  4.1082 -            votable.add_column(name="View",data=temp_row_view,position=0)
  4.1083 -            votable.add_column(name="Img",data=temp_row_img,position=1)
  4.1084 -            votable.add_column(name="Cat",data=temp_row_cat,position=2)
  4.1085 -
  4.1086 -            cols = list(votable.columns)
  4.1087 -            
  4.1088 -            lst = []
  4.1089 -            for x in votable.data.tolist():
  4.1090 -                second_counter = 0
  4.1091 -                obj_list = {}
  4.1092 -                for y in x:
  4.1093 -                    obj_list[cols[second_counter]] = y 
  4.1094 -                    second_counter = second_counter + 1
  4.1095 -                lst.append(obj_list)
  4.1096 -            
  4.1097 -            #html_functions.escape_list(votable.data.tolist())
  4.1098 -            final_lst = [cols,lst] 
  4.1099 -            file_handle = File_handler()
  4.1100 -            file_handle.create_temp_file()
  4.1101 -            file_handle.write_to_temp_file(json.dumps(final_lst))
  4.1102 -            file_handle.close_handle()    
  4.1103 -            file_path = file_handle.pathname
  4.1104 -
  4.1105 -           
  4.1106 -    except Exception as e:
  4.1107 -        logging.exception('Exception caught:')
  4.1108 -    return (votable,"",file_path,"","")
  4.1109 -   
  4.1110 -
  4.1111 - 
  4.1112 -def write_format_from_json(to_format,json_list, visible_cols=None):
  4.1113 -    """
  4.1114 -    Generates HTML to_format for a json array of data
  4.1115 -    
  4.1116 -    @param to_format: The format to which to transform a JSON list of data
  4.1117 -    @param json_list: The source json list
  4.1118 -    """
  4.1119 -
  4.1120 -    html_array = json_list
  4.1121 -    fields = html_array[0] if visible_cols == None else visible_cols
  4.1122 -    data = html_array[1]
  4.1123 -    t = atpy.Table()
  4.1124 -    
  4.1125 -    if to_format == 'csv':
  4.1126 -   
  4.1127 -        csv = ''
  4.1128 -        counter = 0 
  4.1129 -        if visible_cols != None:
  4.1130 -            for col in fields:
  4.1131 -                counter = counter + 1
  4.1132 -                if counter == len(fields):
  4.1133 -                    csv += str(col)
  4.1134 -                else:
  4.1135 -                    csv += str(col) + ','
  4.1136 -       
  4.1137 -        csv += '\n'
  4.1138 -       
  4.1139 -        for row in data:
  4.1140 -            values = []
  4.1141 -            
  4.1142 -            for col in fields:
  4.1143 -                try:   	
  4.1144 -	       	       
  4.1145 -	       	        if (type(row[col])!=unicode):
  4.1146 -	       	            values.append(str(row[col]))
  4.1147 -	       	        else:
  4.1148 -	       	            values.append('"' + str(row[col]) + '"')
  4.1149 -                    
  4.1150 -                except Exception as e:
  4.1151 -                    logging.exception(e)
  4.1152 -		
  4.1153 -            csv += ",".join(values) 
  4.1154 -            csv += '\n'
  4.1155 -            
  4.1156 -        result = csv
  4.1157 -    
  4.1158 -    else:    
  4.1159 -        
  4.1160 -        for val in fields:
  4.1161 -            
  4.1162 -            col_data = []
  4.1163 -            for row in data:
  4.1164 -             
  4.1165 -                if(type(row[val])==unicode):
  4.1166 -                    col_data.append(numpy.str(row[val]))
  4.1167 -                    mytype = numpy.string_
  4.1168 -                else:
  4.1169 -                    col_data.append(row[val])
  4.1170 -                    mytype = type(row[val])
  4.1171 -            try:
  4.1172 -                t.add_column(str(val),numpy.array(col_data),dtype=mytype)
  4.1173 -            except Exception as e:
  4.1174 -                t.add_column(str(val),numpy.array(col_data),dtype=numpy.string_)
  4.1175 -    
  4.1176 -        d = StringIO.StringIO()
  4.1177 -        t.write(d, type=to_format)
  4.1178 -        result = d.getvalue()
  4.1179 -        d.close()
  4.1180 -
  4.1181 -    
  4.1182 -    return result            
  4.1183 -
  4.1184 -
  4.1185 -
  4.1186 -def generate_query_add_secondary_order(query, colid):
  4.1187 -    """
  4.1188 -    Add a secondary order clause to an sql query to help with data consistency when data from query returned has the same value
  4.1189 -    """
  4.1190 -    query_as_string = query
  4.1191 -    if query.lower().find(" order ")>0:
  4.1192 -        query = query.replace('(', ' ( ')
  4.1193 -        query = query.replace(')', ' ) ')
  4.1194 -        query = query.replace(',', ' , ')
  4.1195 -        query_list = query.split(" ")
  4.1196 -        query_list = filter(None, query_list)
  4.1197 -        new_query_list = []
  4.1198 -        prev_item = ""
  4.1199 -        prev_prev_item = ""
  4.1200 -        secondary_order = "," + colid + " "
  4.1201 -        is_order_expression = False
  4.1202 -        insert_next = False
  4.1203 -
  4.1204 -        for item in query_list:
  4.1205 -            if prev_item.lower() == "by" and prev_prev_item.lower() == "order":
  4.1206 -                is_order_expression = True
  4.1207 -            
  4.1208 -            if is_order_expression == True:
  4.1209 -                if insert_next == True and item!="," and item.find(',')<0: 
  4.1210 -                    new_query_list.append(secondary_order)
  4.1211 -                    new_query_list.append(item + " ")
  4.1212 -                    insert_next=False
  4.1213 -                    is_order_expression = False
  4.1214 -                elif item!="," and item!="" and item.find(',')<0 :
  4.1215 -                    insert_next = True 
  4.1216 -                    new_query_list.append(item + " ")
  4.1217 -                else :
  4.1218 -                    insert_next=False
  4.1219 -                    new_query_list.append(item + " ")
  4.1220 -                 
  4.1221 -            else:
  4.1222 -                new_query_list.append(item + " ")
  4.1223 -            
  4.1224 -          
  4.1225 -                                                
  4.1226 -            prev_prev_item = prev_item
  4.1227 -            prev_item = item
  4.1228 -        
  4.1229 -        if is_order_expression == True:
  4.1230 -            new_query_list.append(secondary_order)
  4.1231 -
  4.1232 -        query_as_string = "".join(new_query_list)
  4.1233 -    
  4.1234 -    else :
  4.1235 -        query_as_string = query 
  4.1236 -        query_as_string += " order by " + colid 
  4.1237 -    return query_as_string
  4.1238 -    
  4.1239 -    
  4.1240 -def get_parent_workspace(schema):
  4.1241 -    """
  4.1242 -    Get the parent workspace of a schema
  4.1243 -    """
  4.1244 -    
  4.1245 -    req = urllib2.Request( schema, headers={"Accept" : "application/json", "firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : session.get("community_input","public (unknown)")} )
  4.1246 -    response = urllib2.urlopen(req)
  4.1247 -    workspace = json.loads(response.read())["parent"]
  4.1248 -    response.close()
  4.1249 -    return workspace
  4.1250 -
  4.1251 -
  4.1252 -def get_info(obj):
  4.1253 -    """
  4.1254 -    Get the parent workspace of a schema
  4.1255 -    """
  4.1256 -    
  4.1257 -    req = urllib2.Request( obj, headers={"Accept" : "application/json", "firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : session.get("community_input","public (unknown)")} )
  4.1258 -    response = urllib2.urlopen(req)
  4.1259 -    info = json.loads(response.read())
  4.1260 -    response.close()
  4.1261 -    return info