clearwing

changeset 14:cff6dbebe033 1.1.1

v1.1
author Stelios Voutsinas <stv@roe.ac.uk>
date Sun Feb 09 23:48:43 2014 +0000 (2014-02-09)
parents 4df973a046de
children 68881bbc0a3f
files src/freeform_sql/misc.py src/static/static_vo_tool/js/freeformJS.js src/templates/dbaccess_SQL_form.html src/templates/viewer.html src/url_classes/data_tables_processing.py src/url_classes/helper_urls.pyc src/url_classes/misc.py src/url_classes/viewer.py
line diff
     1.1 --- a/src/freeform_sql/misc.py	Sun Feb 09 23:42:28 2014 +0000
     1.2 +++ b/src/freeform_sql/misc.py	Sun Feb 09 23:48:43 2014 +0000
     1.3 @@ -23,13 +23,13 @@
     1.4  from file_handler import File_handler
     1.5  from helper_functions.string_functions import string_functions
     1.6  string_functions = string_functions()
     1.7 -from config import survey_prefix, SURVEY_DB,base_location, debug_mode, MAX_DELAY, MAX_ELAPSED_TIME, MIN_ELAPSED_TIME_BEFORE_REDUCE, INITIAL_DELAY
     1.8 +from config import survey_prefix,SURVEY_DB,base_location, debug_mode
     1.9  from survey_globals import archive_input
    1.10  from globals import MyOutputStream, logging
    1.11  import datetime
    1.12  import firethorn_config
    1.13  from app import session
    1.14 -from time import gmtime,  strftime
    1.15 +
    1.16  
    1.17  def clear_temp_folder():
    1.18      """
    1.19 @@ -389,35 +389,26 @@
    1.20     
    1.21      f_read = ""
    1.22      return_vot = ''
    1.23 -    delay = INITIAL_DELAY
    1.24 +    delay = 2
    1.25      start_time = datetime.datetime.now()
    1.26      elapsed_time = 0
    1.27      query_json = {'syntax' : {'friendly' : 'A problem occurred while running your query', 'status' : 'Error' }}
    1.28      
    1.29      try:
    1.30 -        logging.exception("Started Firethorn job :::" +  strftime("%Y-%m-%d %H:%M:%S", gmtime()))
    1.31 -
    1.32 +     
    1.33          data = urllib.urlencode({ firethorn_config.query_status_update : "RUNNING"})
    1.34          request = urllib2.Request(url, data, headers={"Accept" : "application/json", "firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : session.get("community_input","public (unknown)")})
    1.35          f_update = urllib2.urlopen(request)
    1.36          query_json =  json.loads(f_update.read())
    1.37          query_status = query_json["status"]
    1.38          
    1.39 -        logging.exception("delay_initial: " + str(delay))
    1.40 -
    1.41 -        while query_status=="PENDING" or query_status=="RUNNING" and elapsed_time<MAX_ELAPSED_TIME:
    1.42 +        
    1.43 +        while query_status=="PENDING" or query_status=="RUNNING" and elapsed_time<18000:
    1.44              query_json = get_status(url)
    1.45              query_status= json.loads(query_json)["status"] 
    1.46              time.sleep(delay)
    1.47 -            if elapsed_time>MIN_ELAPSED_TIME_BEFORE_REDUCE and delay<MAX_DELAY:
    1.48 -                delay = delay + delay
    1.49              elapsed_time = (datetime.datetime.now() - start_time).seconds
    1.50 -            logging.exception("elapsed_time: ")
    1.51 -            logging.exception(elapsed_time)
    1.52 -            logging.exception("delay: " + str(delay))
    1.53 -            
    1.54 -        logging.exception("Completed Firethorn job :::" +  strftime("%Y-%m-%d %H:%M:%S", gmtime()))
    1.55 -  
    1.56 +       
    1.57          if query_status=="ERROR" or query_status=="FAILED":
    1.58              return {'Code' :-1,  'Content' : 'Query error: A problem occurred while running your query' }
    1.59          elif query_status=="CANCELLED":
    1.60 @@ -425,7 +416,7 @@
    1.61          elif query_status=="EDITING":
    1.62              return {'Code' :-1,  'Content' : 'Query error: ' + query_json["syntax"]["status"] + ' - ' + query_json["syntax"]["friendly"] }
    1.63          elif query_status=="COMPLETED":
    1.64 -            return {'Code' :1,  'Content' : url + '/datatable' }
    1.65 +            return {'Code' :1,  'Content' : url + '/votable' }
    1.66          else :
    1.67              return {'Code' :-1,  'Content' : 'Query error: A problem occurred while running your query' }
    1.68          
    1.69 @@ -550,27 +541,23 @@
    1.70      now = datetime.datetime.now()
    1.71      adql_table=''
    1.72      query_loop_results=''
    1.73 -    cols = []
    1.74 -    rows = []
    1.75      
    1.76      try:
    1.77  
    1.78          # Return results as a votable object
    1.79 -        logging.exception("Started run_query ::::::" +  strftime("%Y-%m-%d %H:%M:%S", gmtime()))
    1.80 +       
    1.81          votable, adql_table, query_loop_results = run_query(q,"",query_space=url)
    1.82 -        if votable!='' and votable !="MAX_ERROR" and votable !="ERROR" and votable!=None:
    1.83 +       
    1.84 +        if votable!='' and votable !="MAX_ERROR" and votable !="ERROR" and getattr(votable, "columns", None)!=None:
    1.85              
    1.86             
    1.87              clear_temp_folder()
    1.88 -           
    1.89 -            #cols = list(votable.columns)
    1.90 -            '''
    1.91 +            cols = list(votable.columns)
    1.92 +            
    1.93              lst = []
    1.94              
    1.95 -            #rows = votable.data.tolist()
    1.96 -            #row_length = len(rows)
    1.97 -            
    1.98 -            
    1.99 +            rows = votable.data.tolist()
   1.100 +            row_length = len(rows)
   1.101              for x in rows:
   1.102                  second_counter = 0
   1.103                  obj_list = {}
   1.104 @@ -580,11 +567,9 @@
   1.105                      second_counter = second_counter + 1
   1.106                  lst.append(obj_list)
   1.107              lst = [cols,lst] 
   1.108 -            '''
   1.109 -            
   1.110              file_handle = File_handler()
   1.111              file_handle.create_temp_file()
   1.112 -            file_handle.write_to_temp_file(votable)
   1.113 +            file_handle.write_to_temp_file(json.dumps(lst))
   1.114              file_handle.close_handle()    
   1.115              file_path = file_handle.pathname
   1.116              
   1.117 @@ -596,7 +581,7 @@
   1.118          if debug_mode != True:
   1.119              from web import ctx
   1.120             
   1.121 -            if votable=="" or votable==None or votable ==[] :
   1.122 +            if votable=="" or votable==None or votable ==[] or getattr(votable, "columns", None)==None:
   1.123                  userip =  "None" if str(web.ctx.ip)==None  else  str(web.ctx.ip)
   1.124                  dbrelease = "Unknown" if url==None else url
   1.125                  rows_error = -1
   1.126 @@ -608,12 +593,11 @@
   1.127                  userip =  "None" if str(web.ctx.ip)==None  else  str(web.ctx.ip)
   1.128                  dbrelease = "Unknown" if url==None else url
   1.129      
   1.130 -                MyOutputStream().log_queries(now.strftime('%Y-%m-%d %H:%M:%S.%f')[:-4], int((datetime.datetime.now() - now).total_seconds()), q, len(rows),len(cols), 
   1.131 +                MyOutputStream().log_queries(now.strftime('%Y-%m-%d %H:%M:%S.%f')[:-4], int((datetime.datetime.now() - now).total_seconds()), q, len(rows),len(votable.columns), 
   1.132                                               html_functions.escape(session.get("username","unknown user")), html_functions.escape(dbrelease) , userip)
   1.133      except Exception:
   1.134          logging.exception('Exception caught:')
   1.135  
   1.136 -    logging.exception("Completed run_query :::" +  strftime("%Y-%m-%d %H:%M:%S", gmtime()))
   1.137  
   1.138      return (votable,jobId,file_path,adql_table, query_loop_results)
   1.139  
   1.140 @@ -641,11 +625,15 @@
   1.141       
   1.142          data = urllib.urlencode({ firethorn_config.query_name_param : query_name,  firethorn_config.query_param : query})
   1.143          request = urllib2.Request(query_space + firethorn_config.query_create_uri, data, headers={"Accept" : "application/json", "firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : session.get("community_input","public (unknown)")})
   1.144 +	
   1.145  
   1.146 +	logging.exception(query)
   1.147  
   1.148          f = urllib2.urlopen(request)
   1.149          query_create_result = json.loads(f.read())
   1.150          query_identity = query_create_result["ident"]
   1.151 +	logging.exception(query_identity)
   1.152 +
   1.153          query_loop_results = start_query_loop(query_identity)
   1.154          results_adql_url = query_create_result["results"]["adql"]
   1.155          
   1.156 @@ -664,12 +652,12 @@
   1.157              elif query_loop_results.get("Code", "") ==1:
   1.158                  req = urllib2.Request(query_loop_results.get("Content", ""), headers={"firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : session.get("community_input","public (unknown)")})
   1.159                  f = urllib2.urlopen(req)
   1.160 -                datatable = f.read(MAX_FILE_SIZE) 
   1.161 +                f.read(MAX_FILE_SIZE) #100Mb = 104,857,600
   1.162               
   1.163                  if len(f.read())>0:
   1.164                      max_size_exceeded = True
   1.165                  if not max_size_exceeded:
   1.166 -                    result = datatable
   1.167 +                    result = atpy.Table(query_loop_results.get("Content", ""), type='vo')
   1.168                  else:
   1.169                      result = ("MAX ERROR",result_adql_table,query_identity)
   1.170               
   1.171 @@ -681,6 +669,7 @@
   1.172          logging.exception('Exception caught:')
   1.173          
   1.174          return ("ERROR",result_adql_table,query_identity)
   1.175 +    
   1.176      if f!='':
   1.177          f.close()
   1.178      return (result,result_adql_table,query_identity)
   1.179 @@ -698,47 +687,35 @@
   1.180      """
   1.181      
   1.182      data_results = ""
   1.183 +    json_list = ""
   1.184      content = ""
   1.185 -    json_data = json.loads(votable)
   1.186 -    
   1.187 -    if len(json_data)<2:
   1.188 -        votable = "ERROR"
   1.189 -    else :
   1.190 -        cols = json_data[0]
   1.191 -        row_length = len(json_data[1])
   1.192 -    
   1.193 -    
   1.194 -    logging.exception("Starting generate_JSON_from_query :::" +  strftime("%Y-%m-%d %H:%M:%S", gmtime()))
   1.195  
   1.196 -    if votable == "MAX_ERROR" or votable == "ERROR":
   1.197 -        data_results = ["ERROR"]	
   1.198 -        if votable == "MAX_ERROR":
   1.199 -            data_results.append("Query exceeded maximum return size limit")
   1.200 -        elif votable == "ERROR":
   1.201 -            data_results.append("There was an error processing your request")
   1.202 -        else:	
   1.203 -            data_results.append(votable)    
   1.204 -    
   1.205 -    
   1.206 -        if query_results_id!="" and query_results_id!=None:
   1.207 -            query_id_content = '<div style="height:33px;text-align:left;margin-left:20px;font-size:13px;color:#C2BEAD">Query ID <a id="toggle_query_info">[+]</a><a id="toggle_query_info_help">[?]</a><div id="toggle_query_info_div" style="display:none">'+ query_results_id.split("/")[-1]  + '</div></div><br>' 
   1.208 -        else :
   1.209 -            query_id_content = ""
   1.210 -        
   1.211 -        data_results.append(query_id_content)
   1.212 -        
   1.213 +    if votable == "MAX_ERROR" or votable == "ERROR" or (getattr(votable, "columns", None)==None and (votable!='' and votable!= None)):
   1.214 +	data_results = ["ERROR"]	
   1.215 +	if votable == "MAX_ERROR":
   1.216 +	    data_results.append("Query exceeded maximum return size limit")
   1.217 +	elif votable == "ERROR":
   1.218 +  	    data_results.append("There was an error processing your request")
   1.219 +	else:	
   1.220 +	    data_results.append(votable)    
   1.221 +	if query_results_id!="" and query_results_id!=None:
   1.222 +	    query_id_content = '<div style="height:33px;text-align:left;margin-left:20px;font-size:13px;color:#C2BEAD">Query ID <a id="toggle_query_info">[+]</a><a id="toggle_query_info_help">[?]</a><div id="toggle_query_info_div" style="display:none">'+ query_results_id.split("/")[-1]  + '</div></div><br>' 
   1.223 +	else :
   1.224 +	    query_id_content = ""
   1.225 +	data_results.append(query_id_content)
   1.226 +	
   1.227          return json.dumps(data_results)
   1.228 -    
   1.229      elif votable!='' and votable!= None:
   1.230 -        if row_length>0:
   1.231 -            cols = list(cols)
   1.232 +        if len(votable)>0:
   1.233 +            cols = list(votable.columns)
   1.234 +            row_list = votable.data.tolist() 
   1.235              if tap_endpoint!="" and query!="":
   1.236                  content = '<form class="launch_viewer" style="float:left;z-index:100;position: relative;" action="' + survey_prefix + '/viewer" method="post" target="_blank"><input type="hidden" name="query" value="'+ string_functions.encode(query) +'"/><input type="hidden" name="cols" value="'+ html_functions.escape(json.dumps(cols)) +'"/><input type="hidden" name="filepath" id="temp_file" value="'+ pathname +'"/><input type="hidden" name="tap_endpoint" value="'+ tap_endpoint +'"/>'
   1.237                  content += '<input type="hidden" name="adql_table" id="adql_table" value="'+ adql_table +'"/>'
   1.238 -                #if table_mode == 'interactive':
   1.239 -                #    data_results = html_functions.escape_list(row_list)
   1.240 -                #    json_list = html_functions.escape(json.dumps(data_results))
   1.241 -                #    content += '<input type="hidden" id="hidden_json_results" name="results" value="' + json_list  + '" />'
   1.242 +                if table_mode == 'interactive':
   1.243 +                    data_results = html_functions.escape_list(row_list)
   1.244 +                    json_list = html_functions.escape(json.dumps(data_results))
   1.245 +                    content += '<input type="hidden" id="hidden_json_results" name="results" value="' + json_list  + '" />'
   1.246                  content += '<div style="margin-bottom:5px;text-align:left;margin-left:20px;font-size:13px;color:#C2BEAD">Launch in Plotter<input type="submit" value="" /></form></div><br /><br />'
   1.247                  content += '<div style="height:33px;clear:both;text-align:left;margin-left:20px;font-size:13px;color:#C2BEAD">Query ID <a id="toggle_query_info">[+]</a><a id="toggle_query_info_help">[?]</a><div id="toggle_query_info_div" style="display:none">'+ query_results_id.split("/")[-1]  + '</div></div><br>'
   1.248  
   1.249 @@ -748,31 +725,29 @@
   1.250              content += '<div style="text-align:left;margin-left:15px;float:left;z-index:100;position: relative;clear:both"><img id ="samp_connection" src="' + survey_prefix + '/static/images/red-button.png" height="17" width="17" /><button id="register">Connect to SAMP</button><button id="unregister" style="display: none;">Disconnect from SAMP</button><button id="loadvotable" style="display: none;">Broadcast results table</button></div>'
   1.251  
   1.252              data_results = [cols]
   1.253 -            data_results.append([row_length])
   1.254 +            data_results.append([len(row_list)])
   1.255              data_results.append([content])
   1.256 -            logging.exception("Completed generate_JSON_from_query :::" +  strftime("%Y-%m-%d %H:%M:%S", gmtime()))
   1.257 -
   1.258              return json.dumps(data_results)
   1.259          else:
   1.260              return json.dumps([])
   1.261      else:
   1.262  
   1.263 -                 
   1.264 -        data_results = ["ERROR"]
   1.265 -        if votable == "MAX_ERROR":
   1.266 -            data_results.append("Query exceeded maximum return size limit")
   1.267 -        elif votable == "ERROR":
   1.268 -            data_results.append("There was an error processing your request")
   1.269 -        else:	
   1.270 -            data_results.append(votable)    
   1.271 -        
   1.272 -        if query_results_id!="" and query_results_id!=None:
   1.273 -            query_id_content = '<div style="height:33px;text-align:left;margin-left:20px;font-size:13px;color:#C2BEAD">Query ID <a id="toggle_query_info">[+]</a><a id="toggle_query_info_help">[?]</a><div id="toggle_query_info_div" style="display:none"> '+ query_results_id.split("/")[-1]  + '</div></div><br>' 
   1.274 -        else :
   1.275 -            query_id_content = ""
   1.276 -        
   1.277 -        data_results.append(query_id_content)     
   1.278 -        return data_results
   1.279 +           
   1.280 +	data_results = ["ERROR"]
   1.281 +	if votable == "MAX_ERROR":
   1.282 +	    data_results.append("Query exceeded maximum return size limit")
   1.283 +	elif votable == "ERROR":
   1.284 +  	    data_results.append("There was an error processing your request")
   1.285 +	else:	
   1.286 +	    data_results.append(votable)    
   1.287 +
   1.288 +	if query_results_id!="" and query_results_id!=None:
   1.289 +	    query_id_content = '<div style="height:33px;text-align:left;margin-left:20px;font-size:13px;color:#C2BEAD">Query ID <a id="toggle_query_info">[+]</a><a id="toggle_query_info_help">[?]</a><div id="toggle_query_info_div" style="display:none"> '+ query_results_id.split("/")[-1]  + '</div></div><br>' 
   1.290 +	else :
   1.291 +	    query_id_content = ""
   1.292 +
   1.293 +	data_results.append(query_id_content)     
   1.294 +	return data_results
   1.295  
   1.296  
   1.297  
     2.1 --- a/src/static/static_vo_tool/js/freeformJS.js	Sun Feb 09 23:42:28 2014 +0000
     2.2 +++ b/src/static/static_vo_tool/js/freeformJS.js	Sun Feb 09 23:48:43 2014 +0000
     2.3 @@ -1243,16 +1243,16 @@
     2.4  	
     2.5  							         	     	$('body').on('click','#facebox .delete, #facebox .add', function(e) {
     2.6  							         	     	    e.preventDefault();
     2.7 -							         	     		item_a = $(this).parent();
     2.8 -							         	     		console.log(item_a);
     2.9 +							         	     		_item = $(this).parent();
    2.10  
    2.11 -							         	     	   $( item_a).fadeOut(function() {
    2.12 +
    2.13 +							         	     	   $( _item).fadeOut(function() {
    2.14  							         	     	    
    2.15 -							         	     	        if (item_a.parent().attr('id') == 'unselected_catalogues') {
    2.16 +							         	     	        if (_item.parent().attr('id') == 'unselected_catalogues') {
    2.17  							         	     	        
    2.18 -							         	     	            $('#facebox #selected_catalogues').append(item_a.fadeIn());
    2.19 +							         	     	            $('#facebox #selected_catalogues').append(_item.fadeIn());
    2.20  							         	     	        } else {
    2.21 -							         	     	            $('#facebox #unselected_catalogues').append(item_a.fadeIn());
    2.22 +							         	     	            $('#facebox #unselected_catalogues').append(_item.fadeIn());
    2.23  							         	     	        }
    2.24  							         	     	    });
    2.25  							         	     	    return false;
     3.1 --- a/src/templates/dbaccess_SQL_form.html	Sun Feb 09 23:42:28 2014 +0000
     3.2 +++ b/src/templates/dbaccess_SQL_form.html	Sun Feb 09 23:48:43 2014 +0000
     3.3 @@ -22,9 +22,6 @@
     3.4  
     3.5  
     3.6  
     3.7 -
     3.8 -
     3.9 -
    3.10  <div id="content_dbaccess_SQL_form">
    3.11      <h2>Freeform Query</h2>
    3.12  	This form allows you to submit a query to the OSA database ( <a
    3.13 @@ -54,7 +51,7 @@
    3.14  	             $:form.render() 
    3.15  	             
    3.16  	             <p style="font-size:11px;margin-left:45px;font-style:italic;color:#81F781;width:90%">
    3.17 -	         		 Press ctrl-space for autocompletion (Load the Table Metadata to include tables)
    3.18 +	         		 Press ctrl-space for SQL autocompletion
    3.19  	          
    3.20  	      		  </p>
    3.21  	      		 
     4.1 --- a/src/templates/viewer.html	Sun Feb 09 23:42:28 2014 +0000
     4.2 +++ b/src/templates/viewer.html	Sun Feb 09 23:48:43 2014 +0000
     4.3 @@ -1,4 +1,4 @@
     4.4 -$def with (text, query, tap_endpoint, filepath,survey_prefix, rand)
     4.5 +$def with (text, query, tap_endpoint, filepath, results, survey_prefix)
     4.6  
     4.7  <!DOCTYPE html PUBLIC"-//W3C//DTD XHTML 1.0 Strict//EN"
     4.8  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
     4.9 @@ -85,7 +85,6 @@
    4.10  	type="text/javascript" charset="utf-8"></script>
    4.11  <script type="text/javascript">			
    4.12          
    4.13 -
    4.14          /*
    4.15           * WFAU Web2.0 Project
    4.16           * Author: Stelios Voutsinas
    4.17 @@ -298,7 +297,7 @@
    4.18      	   
    4.19            jQuery(document).ready(function(){
    4.20          	
    4.21 -        	  jQuery('a[rel*=facebox]').facebox();  // Initialize facebox links 
    4.22 +        	  
    4.23          	   /* Samp */
    4.24         
    4.25          	    cc = new samp.ClientTracker();
    4.26 @@ -585,7 +584,7 @@
    4.27  			 
    4.28          	 /* end samp */
    4.29          	  
    4.30 -         	
    4.31 +         	jQuery('a[rel*=facebox]').facebox();  // Initialize facebox links 
    4.32  
    4.33   
    4.34          	// Variables used for scatter plot 
    4.35 @@ -957,10 +956,10 @@
    4.36  
    4.37                	// Draw the individual points in areas with low density
    4.38                	for (var i = 0; i < sparsePts.length; i++) {
    4.39 -              	
    4.40 +              		
    4.41                		var xPos = chart.xAxis[0].translate(sparsePts[i][0])+chart.plotLeft;
    4.42                		var yPos = chart.plotHeight-chart.yAxis[0].translate(sparsePts[i][1])+chart.plotTop;
    4.43 -              			 primitiveObjects.push(chart.renderer.circle(xPos,yPos,3).attr({dataX : sparsePts[i][0], dataY : sparsePts[i][1], selected : 'False', fill: 'rgba(119, 152, 191, .7)',zIndex: 100}).on('click', function(){
    4.44 +              	    primitiveObjects.push(chart.renderer.circle(xPos,yPos,3).attr({dataX : sparsePts[i][0], dataY : sparsePts[i][1], selected : 'False', fill: 'rgba(119, 152, 191, .7)',zIndex: 100}).on('click', function(){
    4.45        					 	
    4.46  		              		var selected = jQuery(this)[0].attributes.selected.value;
    4.47        					 	if (selected=='True'){
    4.48 @@ -968,7 +967,7 @@
    4.49    	     				    		 fnResetAllFilters(oTable);
    4.50    	     				    		 jQuery(this)[0].attributes.selected.value = 'False';
    4.51    	     				    		 jQuery(this)[0].attributes['fill-opacity'].value = '0.6';
    4.52 -									jQuery(this)[0].attributes['fill'].value ='rgb(119,152,191)';
    4.53 +							jQuery(this)[0].attributes['fill'].value ='rgb(119,152,191)';
    4.54        					 	 } else {
    4.55        					 		 resetPointSelections();
    4.56  				        	     var x = jQuery(this)[0].attributes.dataX.value;
    4.57 @@ -1260,21 +1259,19 @@
    4.58          		fnResetAllFilters(oTable);
    4.59          		resetPointSelections();
    4.60      	    });  
    4.61 -        
    4.62 -       	    
    4.63 +       	  
    4.64              /** 
    4.65               * On plot expression link click, toggle the post-it example div 
    4.66      		 *
    4.67      		 */
    4.68 -            jQuery('#plot_expression_help_link').live('click',function(e){ 
    4.69 -            	e.preventDefault();
    4.70 +            jQuery('#plot_expression_help_link').live('click',function(){ 
    4.71      		  	if (jQuery('#plot_expression_help_link').html() == '[+]'){ 
    4.72  			  		jQuery('#plot_expression_help_link').html('[-]');
    4.73  			  	} else {
    4.74  			  		jQuery('#plot_expression_help_link').html('[+]');
    4.75  			  	}
    4.76      			jQuery('#post-it-example').slideToggle('slow');
    4.77 -			  	return false;
    4.78 +			  
    4.79        	    });
    4.80         	    
    4.81              /**
    4.82 @@ -1304,18 +1301,6 @@
    4.83      			}
    4.84         	    }); 
    4.85         	    
    4.86 -       	    jQuery('#x_axis').live('change',function(){ 
    4.87 -				
    4.88 -				if (jQuery("#x_alias").val()=="" || jQuery("#x_alias").val()==null )
    4.89 -					jQuery("#x_alias").val(jQuery("#x_axis").val());
    4.90 -       	    });
    4.91 -	       	
    4.92 -       	    jQuery('#y_axis').live('change',function(){ 
    4.93 -				
    4.94 -				if (jQuery("#y_alias").val()=="" || jQuery("#y_alias").val()==null )
    4.95 -					jQuery("#y_alias").val(jQuery("#y_axis").val());
    4.96 -    	    });
    4.97 -            
    4.98         	    /** 
    4.99         	     * On Plot Type change, adjust inputs displayed 
   4.100         	     *
   4.101 @@ -1415,7 +1400,7 @@
   4.102         					if (tables.length>0) {
   4.103  
   4.104         						var filteredArray = jQuery.map(tags, function(item) {
   4.105 -	       						if( item.indexOf( extractLast(request.term))=== 0 ){
   4.106 +	       						if( item.startsWith( extractLast(request.term))){
   4.107  						        	return item;
   4.108  						        } else {
   4.109  						            return null;
   4.110 @@ -1425,8 +1410,7 @@
   4.111         					} else {
   4.112  
   4.113  	       					var filteredArray = jQuery.map(availableTags, function(item) {
   4.114 -
   4.115 -								if(  item.indexOf( extractLast(request.term))=== 0){
   4.116 +								if( item.startsWith( extractLast(request.term))){
   4.117  						        	return item;
   4.118  						        } else {
   4.119  						            return null;
   4.120 @@ -1435,14 +1419,13 @@
   4.121         					}
   4.122         					response( jQuery.ui.autocomplete.filter(
   4.123         							filteredArray, extractLast( request.term ) ) );
   4.124 -					
   4.125         				},
   4.126         				focus: function() {
   4.127         					// prevent value inserted on focus
   4.128         					return false;
   4.129         				},
   4.130         				select: function( event, ui ) {
   4.131 -					var terms = split( this.value);
   4.132 +						var terms = split( this.value);
   4.133  
   4.134         					//var terms = this.value.split(/[\s,]+/);
   4.135         					var new_array = [];
   4.136 @@ -1467,16 +1450,15 @@
   4.137         					// add placeholder to get the comma-and-space at the end
   4.138         					terms.push( "" );
   4.139         					this.value = terms.join( " " );
   4.140 -					
   4.141 -					if (jQuery(this).attr("id")=="x_axis"){
   4.142 -						jQuery("#x_alias").val(this.value);
   4.143 -       	    				} else if (jQuery(this).attr("id")=="y_axis"){
   4.144 -						jQuery("#y_alias").val(this.value);
   4.145 +       					
   4.146 +       					if (jQuery(this).attr("id")=="x_axis"){
   4.147 +						    jQuery("#x_alias").val(this.value);
   4.148 +       	    			} else if (jQuery(this).attr("id")=="y_axis"){
   4.149 +						    jQuery("#y_alias").val(this.value);
   4.150         	    				
   4.151 -					}
   4.152 +					    }
   4.153  						
   4.154  
   4.155 -
   4.156         					return false;
   4.157         				}
   4.158         			
   4.159 @@ -1586,12 +1568,9 @@
   4.160  												            jQuery('#results').fadeIn('slow');
   4.161  	  								             		    jQuery("#results .scroll-wrapper").remove();
   4.162  										                    jQuery('#results .dataTables_scrollBody').doubleScroll();
   4.163 - var row_length = oTable.fnSettings().fnRecordsTotal();
   4.164 - if (row_length>25000){  
   4.165 - 	jQuery("#length_notify").html("Plotting " + row_length + " rows with interactive plotting may be slow. We suggest you try static plotting instead" ).fadeIn();
   4.166 -}						               	}
   4.167 +
   4.168 +									               	}
   4.169  								                 }
   4.170 -
   4.171  								              }); //end ajax call
   4.172  											 
   4.173  							 } //end fnServerData function	            
   4.174 @@ -1599,13 +1578,12 @@
   4.175  			            } ); //end oTable
   4.176  
   4.177  		            }
   4.178 -			        oTable.fnAdjustColumnSizing(true);     
   4.179 -			       		               
   4.180 +			        oTable.fnAdjustColumnSizing(true);                       
   4.181  				}
   4.182         		} catch (ex){
   4.183         			alert ('There was an error processing you request');
   4.184         		}
   4.185 -
   4.186 +       		
   4.187         		
   4.188         		
   4.189              /** 
   4.190 @@ -1714,12 +1692,7 @@
   4.191                          },
   4.192                          success: function(data) {
   4.193                  			jQuery('#load').hide();  
   4.194 -                			if (data=="ERROR"){
   4.195 -                				jQuery('#error').html("There was a server error while processing your request");
   4.196 -	        					jQuery('#error').fadeIn('normal');
   4.197 -	        					jQuery('#error').delay(3800).fadeOut('slow');        
   4.198 -	        					result_array = "error";
   4.199 -                			} else if (data!="ERROR"){	                   
   4.200 +                			if (data!="<div style=\"color:red;margin-left:50px;font-size:12px\">There was an error processing your request</div>"){	                   
   4.201                  	     		jQuery('#viewer_table_div').hide();	
   4.202                  	     		jQuery('#user_table_toggle').html('[+]');
   4.203                  				jQuery('#viewer_table_div .generated_table_content').html('');
   4.204 @@ -2381,7 +2354,7 @@
   4.205  	        		}
   4.206  	        		
   4.207  	        	 	params.push({'name' : '_broadcast_url', 'value' : _broadcast_url});
   4.208 -				params.push({'name' : 'getRowIndexes', 'value' : 'true'}); 
   4.209 +					params.push({'name' : 'getRowIndexes', 'value' : 'true'}); 
   4.210  	        	 	params.push({'name' : 'pathname', 'value' : pathname});
   4.211  	    			
   4.212  	        	 	var counter = 0;
   4.213 @@ -2426,10 +2399,10 @@
   4.214  						var axis_names = jQuery("#container div.highcharts-container svg text.highcharts-title tspan").text();
   4.215  						if (axis_names.trim()!="Table-Plotter Interactive Histogram Plot" && jQuery("#mode_notify").html()=="interactive"){
   4.216  			        		axis_names = axis_names.split('  ');
   4.217 -						var x_axis = axis_names[0];
   4.218 -						var y_axis = axis_names[2];
   4.219 -						var x_axis_index = -1;
   4.220 -						var y_axis_index = -1;
   4.221 +							var x_axis = axis_names[0];
   4.222 +							var y_axis = axis_names[2];
   4.223 +							var x_axis_index = -1;
   4.224 +							var y_axis_index = -1;
   4.225  			        		var oSettings = oTable.fnSettings();  
   4.226  			        		var is_math_expr = jQuery('#is_math_expr').val();
   4.227  			        		var circle = [];
   4.228 @@ -2472,7 +2445,7 @@
   4.229  			     
   4.230  			        	 	jQuery.each(jQuery('circle'), function(i, val) {
   4.231  			        	    	 if (jQuery(this)[0].attributes.selected.value == 'True'){
   4.232 -								 jQuery('#point_notification').slideUp('slow');	 
   4.233 +									 jQuery('#point_notification').slideUp('slow');	 
   4.234  						    		 jQuery(this)[0].attributes.selected.value = 'False';
   4.235  						    		 jQuery(this)[0].attributes['fill-opacity'].value = '0.6';
   4.236  								 jQuery(this)[0].attributes['fill'].value ='rgb(119,152,191)';	
   4.237 @@ -2637,12 +2610,12 @@
   4.238  					id="filepath" $:filepath />
   4.239  				<input type="hidden" style="display: none" name="tap_endpoint"
   4.240  					id="tap_endpoint" $:tap_endpoint />
   4.241 +				<input type="hidden" style="display: none" name="results" $:results />
   4.242  				<input type="hidden" style="display: none" name="is_math_expr"
   4.243  					id="is_math_expr" value="False" />
   4.244  
   4.245  				<div style="width: 570px; margin: 10px; height: 30px;">
   4.246 -					<label class="short_label" for="plot_type">Plot type:</label><a href="#help_plot_type" rel="facebox"
   4.247 -				style="color: gray; text-decoration: none;">[?]</a> <select
   4.248 +					<label class="short_label" for="plot_type">Plot type:</label> <select
   4.249  						name="plot_type" id="plot_type"
   4.250  						style="margin-top: 5px; margin-left: 10px;">
   4.251  						<option value="Mixed">Mixed Scatter/Density</option>
   4.252 @@ -2708,7 +2681,7 @@
   4.253  		style="width: 900px; min-height: 620px; margin: 0 auto; display: none; margin-bottom: 5px"></div>
   4.254  
   4.255  	<div id="load"></div>
   4.256 -	<div id="length_notify" style="display:none;color:green;margin:0 auto;text-align:center;font-style:italic;margin-top:5px;"></div>
   4.257 +
   4.258  	<div id="density_notification"
   4.259  		style="display: none; font-size: 11px; color: green; margin: 0 auto; text-align: center; font-weight: bold"></div>
   4.260  
   4.261 @@ -2763,7 +2736,7 @@
   4.262  		</div>
   4.263  
   4.264  		<div
   4.265 -			style="margin-left: 80px; margin-top: 18px; float: left; z-index: 45; position: relative;">
   4.266 +			style="margin-left: 80px; margin-top: 18px; float: left; z-index: 100; position: relative;">
   4.267  			<img id="samp_connection"
   4.268  				src="$survey_prefix/static/images/red-button.png" height="17"
   4.269  				width="17" />
   4.270 @@ -2776,31 +2749,10 @@
   4.271  
   4.272  		<div id="results"></div> <br>
   4.273  
   4.274 -			<div id="help_plot_type" style="display: none; font-size: 13px;">
   4.275 -				<h2>Help:</h2>
   4.276 +			<div id="help" style="display: none; font-size: 13px;">
   4.277 +				<h5>Help:</h5>
   4.278  				<br><br>
   4.279  						<p style="font-size: 12px;">
   4.280 -							<h4>Interactive mode</h4><hr>
   4.281 -							<b>Mixed Scatter/Density plot:</b> A scatter plot where dense areas will be grouped into bins
   4.282 -							, the color of which depends on the density (number of points in that area)<br />
   4.283 -							<b>Scatter plot:</b> A simple scatter plot without any binning. (Preferable for plotting a small number of points)<br />
   4.284 -							<b>Histogram:</b> A Histogram plot with the x axis being ranges for the given column value or expression, and the y-axis being the according count<br />
   4.285 -						</p>
   4.286 -						<br><br>	
   4.287 -						<p style="font-size: 12px;">
   4.288 -							<h4>Static mode</h4><hr>
   4.289 -							<b>Density plot:</b> A denisty plot created as a static image with a color bar displaying the density colorization<br />
   4.290 -							<b>Scatter plot:</b> A simple scatter plot created as a static image<br />
   4.291 -							<b>Histogram:</b> A Histogram plot created as a static image<br />
   4.292 -						</p>
   4.293 -						
   4.294 -			</div>
   4.295 -
   4.296 -
   4.297 -			<div id="help" style="display: none; font-size: 13px;">
   4.298 -				<h2>Help:</h2>
   4.299 -				<br/><br/>
   4.300 -						<p style="font-size: 12px;">
   4.301  							<b>Interactive mode:</b> An interactive plot will be generated,
   4.302  							where you can zoom in and out, in a dynamic way.<br> Not
   4.303  								recommended for very large data sets (e.g. > 1,000,000 cells),
   4.304 @@ -2810,7 +2762,7 @@
   4.305  										to handle the amount of data you require to have plotted. <br><br><br><br>
   4.306  						</p>
   4.307  			</div>
   4.308 -			
   4.309 +
   4.310  			<div style="visibility: hidden">
   4.311  				<br /> <a href="http://apycom.com/">link</a><br />
   4.312  			</div>
     5.1 --- a/src/url_classes/data_tables_processing.py	Sun Feb 09 23:42:28 2014 +0000
     5.2 +++ b/src/url_classes/data_tables_processing.py	Sun Feb 09 23:48:43 2014 +0000
     5.3 @@ -770,8 +770,7 @@
     5.4          try:       
     5.5              
     5.6                  f = open(data.pathname, 'r')  
     5.7 -                val = f.read()
     5.8 -                jsn_encoded_stored_data = json.loads(val)
     5.9 +                jsn_encoded_stored_data = json.load(f)
    5.10                  sidx = data.iSortCol_0
    5.11                  start = data.iDisplayStart
    5.12                  rows = data.iDisplayLength
    5.13 @@ -897,9 +896,8 @@
    5.14               
    5.15               
    5.16              
    5.17 -            f = open(data.pathname, 'r')  
    5.18 -            val = f.read()    
    5.19 -            jsn = json.loads(val)
    5.20 +            f = open(data.pathname, 'r')    
    5.21 +            jsn =  json.load(f)
    5.22              sidx = data.iSortCol_0
    5.23              start = data.iDisplayStart
    5.24              
     6.1 Binary file src/url_classes/helper_urls.pyc has changed
     7.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     7.2 +++ b/src/url_classes/misc.py	Sun Feb 09 23:48:43 2014 +0000
     7.3 @@ -0,0 +1,1080 @@
     7.4 +"""
     7.5 +misc module
     7.6 +
     7.7 +Miscellaneous functions used by for freeform SQL query processing for use with TAP services and XML VOTable procesing
     7.8 +
     7.9 +Created on Nov 30, 2011
    7.10 +@author: stelios
    7.11 +"""
    7.12 +
    7.13 +import os
    7.14 +import urllib2
    7.15 +import urllib
    7.16 +import StringIO
    7.17 +import time
    7.18 +import xml.dom.minidom
    7.19 +import atpy
    7.20 +import json
    7.21 +import numpy
    7.22 +import html_functions
    7.23 +import xml_parser
    7.24 +from freeform_config import *
    7.25 +import re
    7.26 +from file_handler import File_handler
    7.27 +from helper_functions.string_functions import string_functions
    7.28 +string_functions = string_functions()
    7.29 +from config import survey_prefix,SURVEY_DB,base_location, debug_mode
    7.30 +from survey_globals import archive_input
    7.31 +from globals import MyOutputStream, logging
    7.32 +import datetime
    7.33 +import firethorn_config
    7.34 +from app import session
    7.35 +
    7.36 +
    7.37 +def clear_temp_folder():
    7.38 +    """
    7.39 +    Clean the temporary folder used for query data, from files that are older than one day
    7.40 +    
    7.41 +    """
    7.42 +    
    7.43 +    try:    
    7.44 +        path = (cur_static_dir + 'temp/')
    7.45 +        listing = os.listdir(path)
    7.46 +        curtime = time.time()
    7.47 +        for infile in listing:
    7.48 +            temp_file = path + infile
    7.49 +            ftime = os.path.getmtime(temp_file)
    7.50 +            difftime = curtime - ftime
    7.51 +            if difftime > 86400:
    7.52 +                if not os.path.isdir(temp_file):
    7.53 +                    os.unlink(temp_file)
    7.54 +    except Exception as e:
    7.55 +        logging.exception('Exception caught:')
    7.56 +       
    7.57 +       
    7.58 +       
    7.59 +
    7.60 +
    7.61 +def change_destruction_time(url,new_time):
    7.62 +    """ 
    7.63 +    Change the destruction time for a TAP job
    7.64 +
    7.65 +    ATTENTION: Currently unused in this version
    7.66 +    
    7.67 +    @param url: The URL for the TAP job to destroy
    7.68 +    @param new_time : The current time to set for the destruction of the TAP job
    7.69 +    """
    7.70 +    
    7.71 +    f = ''
    7.72 +    try:
    7.73 +        params = urllib.urlencode({'DESTRUCTION' : new_time}) 
    7.74 +        url = url+'/destruction'
    7.75 +        req = urllib2.Request(url, params)
    7.76 +        f = urllib2.urlopen(req)
    7.77 +        f.close()
    7.78 +    except Exception:
    7.79 +        if f!='':
    7.80 +            f.close()
    7.81 +        logging.exception('Exception caught:')
    7.82 +
    7.83 + 
    7.84 + 
    7.85 +
    7.86 +
    7.87 +
    7.88 +def generate_form_data(registry):
    7.89 +    """
    7.90 +    Fetch array of available TAP endpoints from the input registry. Uses XQuery, and returns a tuple of web.form.Checkbox objects, to be added to the tap endpoint selection form
    7.91 +    
    7.92 +    @param registry: A string containin the Registry URL
    7.93 +    @return: An tuple containing the registry endpoints as web.checkbox elements
    7.94 +    """
    7.95 +
    7.96 +    url_query = """
    7.97 +    //RootResource[@status='active' and capability[@standardID='ivo://ivoa.net/std/TAP']]/capability[@standardID='ivo://ivoa.net/std/TAP']/interface/accessURL
    7.98 +    """
    7.99 +
   7.100 +
   7.101 +    title_query = """
   7.102 +    //RootResource[@status='active' and capability[@standardID='ivo://ivoa.net/std/TAP']]/title
   7.103 +    """ 
   7.104 +
   7.105 +  
   7.106 +    form_data=""
   7.107 +    title_results = []
   7.108 +    tap_url_results = []
   7.109 +    f=''
   7.110 +    try:
   7.111 +        # get title elements
   7.112 +        params = urllib.urlencode({'performquery' : 'true', 'ResourceXQuery' : title_query}) 
   7.113 +        req = urllib2.Request(registry, params)
   7.114 +        f = urllib2.urlopen(req)
   7.115 +        result_string = f.read()
   7.116 +       
   7.117 +        RE_XML_ILLEGAL = u'([\u0000-\u0008\u000b-\u000c\u000e-\u001f\ufffe-\uffff])' + \
   7.118 +                 u'|' + \
   7.119 +                 u'([%s-%s][^%s-%s])|([^%s-%s][%s-%s])|([%s-%s]$)|(^[%s-%s])' % \
   7.120 +                  (unichr(0xd800),unichr(0xdbff),unichr(0xdc00),unichr(0xdfff),
   7.121 +                   unichr(0xd800),unichr(0xdbff),unichr(0xdc00),unichr(0xdfff),
   7.122 +                   unichr(0xd800),unichr(0xdbff),unichr(0xdc00),unichr(0xdfff))
   7.123 +        result_string = re.sub(RE_XML_ILLEGAL, "?", result_string)
   7.124 +        
   7.125 +        result_string = result_string.decode('latin-1')
   7.126 +        dom = xml.dom.minidom.parseString(result_string.encode('utf-8'))
   7.127 +        title_results=dom.getElementsByTagName("title")
   7.128 +
   7.129 +        # get TAP urls
   7.130 +        params2 = urllib.urlencode({'performquery' : 'true', 'ResourceXQuery' : url_query}) 
   7.131 +        req2 = urllib2.Request(registry, params2)
   7.132 +        f2 = urllib2.urlopen(req2)
   7.133 +        result_string2 = f2.read()
   7.134 +        dom2 = xml.dom.minidom.parseString(result_string2)
   7.135 +        tap_url_results=dom2.getElementsByTagName("accessURL")
   7.136 +      
   7.137 +        counter = 0
   7.138 +        for i in tap_url_results:
   7.139 +            form_data+='<li><span>' + xml_parser.getText(title_results[counter]) + '</span> as <input type="text" name="catalogue"  value="'  + xml_parser.getText(title_results[counter]) +  '" /><input type="hidden" name="ident" value="' + xml_parser.getText(i) + '"/>' +  remove_catalogue_html + '</li>'
   7.140 +            counter = counter +1
   7.141 +            
   7.142 +        f.close()
   7.143 +        f2.close()
   7.144 +    
   7.145 +    except Exception as e:
   7.146 +        if f!='':
   7.147 +            f.close()   
   7.148 +        logging.exception('Exception caught:')      
   7.149 +        return ""
   7.150 +    if form_data=="":
   7.151 +        return ""
   7.152 +    else:
   7.153 +        return form_data
   7.154 +
   7.155 +
   7.156 +
   7.157 +
   7.158 +
   7.159 +
   7.160 +def generate_endpoint_from_list(endpnt_list):
   7.161 +    """
   7.162 +    Takes an list of TAP endpoint URLs separated by a double space, and generates a merged TAP service from the OGSDA-DAI TAP factory(global tap_factory variable)
   7.163 +           
   7.164 +    @param endpnt_lits: A list of available TAP endpoints as strings
   7.165 +    @param prefixes: A list of prefixes as a json object, from which we extract URL & name and submit to the Registry
   7.166 +    @return: A string with the new URL to be used by the client
   7.167 +    """
   7.168 +    
   7.169 +    data = ""
   7.170 +    result= ""
   7.171 +    urls = endpnt_list
   7.172 +    response = ''
   7.173 +    counter = 0
   7.174 +    query_schema = ''
   7.175 +    importname=""
   7.176 +    
   7.177 +    from datetime import datetime
   7.178 +    t = datetime.now()
   7.179 +   
   7.180 +    try:
   7.181 +       
   7.182 +        ### Create workspace
   7.183 +        workspace_name = 'workspace-' + t.strftime("%y%m%d_%H%M%S") 
   7.184 +        data = urllib.urlencode({firethorn_config.resource_create_name_params['http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-resource-1.0.json'] : workspace_name})
   7.185 +        req = urllib2.Request( firethorn_config.workspace_creator, data, headers={"Accept" : "application/json", "firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : session.get("community_input","public (unknown)")})
   7.186 +        response = urllib2.urlopen(req)
   7.187 +        workspace = json.loads(response.read())["ident"]
   7.188 +        response.close()
   7.189 +        
   7.190 +        ### Create Query Schema 
   7.191 +        data = urllib.urlencode({firethorn_config.resource_create_name_params['http://data.metagrid.co.uk/wfau/firethorn/types/entity/adql-schema-1.0.json'] : "query_schema"})
   7.192 +        req = urllib2.Request( workspace +  firethorn_config.schema_create_uri, data, headers={"Accept" : "application/json", "firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : session.get("community_input","public (unknown)")})
   7.193 +        response = urllib2.urlopen(req) 
   7.194 +        query_schema = json.loads(response.read())["ident"]
   7.195 +        response.close()
   7.196 +        
   7.197 +      
   7.198 +        for i in urls:
   7.199 +            importname = ""
   7.200 +            name = i['name']
   7.201 +            alias = i['alias']
   7.202 +            ident = i['ident']
   7.203 +   
   7.204 +        
   7.205 +            if alias!="":
   7.206 +                importname = alias
   7.207 +            else :
   7.208 +                importname = name
   7.209 +        
   7.210 +            if importname!="":
   7.211 +                data = urllib.urlencode({'urn:adql.copy.depth' : "THIN", firethorn_config.workspace_import_schema_base : ident, firethorn_config.workspace_import_schema_name : importname})
   7.212 +            else :
   7.213 +                data = urllib.urlencode({'urn:adql.copy.depth' : "THIN", firethorn_config.workspace_import_schema_base : ident})
   7.214 +            
   7.215 +          
   7.216 +            req = urllib2.Request( workspace + firethorn_config.workspace_import_uri, data, headers={"Accept" : "application/json", "firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : session.get("community_input","public (unknown)")}) 
   7.217 +            response = urllib2.urlopen(req)
   7.218 +            response.close()
   7.219 +            
   7.220 +            
   7.221 +    except Exception:
   7.222 +        if response!='':
   7.223 +            response.close()
   7.224 +            result = "" 
   7.225 +        logging.exception("IOError")  
   7.226 +    
   7.227 +  
   7.228 +      
   7.229 +    return query_schema
   7.230 +
   7.231 +
   7.232 +
   7.233 +
   7.234 +#######################################
   7.235 +# Note: Not used in Firethorn version #
   7.236 +#######################################
   7.237 +def fetch_metadata(tap_endpoint):
   7.238 +    """
   7.239 +    Takes a TAP endpoint URL, and generates the table metadata. This is done by reading and parsing the XML table information using the xml_dom.minidom library and the xml_parser module
   7.240 +    
   7.241 +    @param tap_endpoint: A string with the TAP endpoint
   7.242 +    @return: HTML content with the metadata for the given URL
   7.243 +    """
   7.244 +    
   7.245 +    return_str = ''
   7.246 +    f = ''
   7.247 +    try:
   7.248 +        f = urllib.urlopen(tap_endpoint)
   7.249 +        dom_str = f.read()
   7.250 +        dom = xml.dom.minidom.parseString(dom_str)
   7.251 +        return_str = xml_parser.handleXML(dom)
   7.252 +        f.close()
   7.253 +    except Exception as e:
   7.254 +        if f!='':
   7.255 +            f.close()
   7.256 +        return_str =  ""   
   7.257 +        logging.exception(e)   
   7.258 +    return return_str
   7.259 +
   7.260 +
   7.261 +     
   7.262 +def get_children_metadata(ident, _type):
   7.263 + 
   7.264 +    import type_helpers
   7.265 +    from datetime import datetime
   7.266 +    
   7.267 +    error_processing = "<div class='error'>Unable to fetch the table metadata</div>"
   7.268 +    return_val = ''
   7.269 +    f= ""
   7.270 +    ident = string_functions.decode(ident)
   7.271 +    _type = string_functions.decode(_type)
   7.272 +    
   7.273 +    try:
   7.274 +        if ident!="" and _type!="":
   7.275 +            if not type_helpers.isColumn(_type):
   7.276 +                ident = ident + firethorn_config.resource_uris[_type]
   7.277 +            request = urllib2.Request(ident, headers={"Accept" : "application/json"})
   7.278 +            f = urllib2.urlopen(request)
   7.279 +            json_data = json.loads(f.read())
   7.280 +           
   7.281 +            if json_data == [] or json_data== None:
   7.282 +                return_val = "No data found"
   7.283 +            else:
   7.284 +                if type_helpers.isRootType(_type):
   7.285 +                    return_val += '<p class="accordion_table_title">Catalogues:</p>'
   7.286 +                if type_helpers.isSchema(_type):
   7.287 +                    return_val += '<p class="accordion_table_title">Tables:</p>'
   7.288 +                
   7.289 +                if type_helpers.isTable(_type):
   7.290 +                                        
   7.291 +                    return_val += '<p class="accordion_table_title">Columns:</p>'
   7.292 +                
   7.293 +                if type_helpers.isColumn(_type):
   7.294 +
   7.295 +                    return_val += "<div>"
   7.296 +                    return_val += "Data Type: " + str(json_data["meta"]["adql"]["type"]) +"</br>"
   7.297 +                    return_val += "Size: " +  str(json_data["meta"]["adql"]["arraySize"])  +"</br>"
   7.298 +                    return_val += "UCD  Type: "  + str(json_data["meta"]["adql"]["ucd"]["type"]) + " - UCD Value: " + str(json_data["meta"]["adql"]["ucd"]["value"]) + "</br>"
   7.299 +                    return_val += "Unit: "  + str(json_data["meta"]["adql"]["units"]) +  "</br>"
   7.300 +                    return_val += "Utype: "  + str(json_data["meta"]["adql"]["utype"]) +  "</br>"
   7.301 +                    return_val += "</br></div>"
   7.302 +                    return_val += "</br>"
   7.303 +                    
   7.304 +                else:
   7.305 +                    
   7.306 +                    for entry in json_data: 
   7.307 +                        
   7.308 +                       
   7.309 +                        if type_helpers.isRootType(_type):
   7.310 +                            converted_dict = dict([(str(k), v) for k, v in entry.items()])
   7.311 +                            
   7.312 +                          
   7.313 +                            if converted_dict["fullname"]!='query_schema':
   7.314 +                                return_val+='<div id="accordion_catalogue">'
   7.315 +                                jdbcname_url  = converted_dict["root"]
   7.316 +                                request = urllib2.Request(jdbcname_url, headers={"Accept" : "application/json"})
   7.317 +                                f = urllib2.urlopen(request)
   7.318 +                                jdbcname_data = json.loads(f.read())
   7.319 +                                jdbcname = jdbcname_data["fullname"]
   7.320 +                                if jdbcname.endswith('.dbo'):
   7.321 +                                    jdbcname = jdbcname[:-4]
   7.322 +                                
   7.323 +                                return_val += '<input type="hidden" name="resource_type" value="' + converted_dict["type"]+ '" /> <input type="hidden" name="resource_ident" value="' + string_functions.encode(converted_dict["ident"]) +'" />'   +  "<div  class='heading'>" + converted_dict["fullname"] + '<img style="float:right;margin-top:5px" src="' + survey_prefix + '/static/static_vo_tool/asc.gif"/></div>'
   7.324 +                                return_val += "<div id='item_description' style='display:none'> Description: "  + html_functions.escape(str(converted_dict["text"])) + ' <br /> Full Database name:' + jdbcname + ' </div>'
   7.325 +                                return_val+='</div>'
   7.326 +                        elif type_helpers.isSchema(_type):
   7.327 +                            return_val+='<div id="accordion_schema">'
   7.328 +                            converted_dict = dict([(str(k), v) for k, v in entry.items()])
   7.329 +                            return_val += '<input type="hidden" name="resource_type" value="' + converted_dict["type"]+ '" /> <input type="hidden" name="resource_ident" value="' + string_functions.encode(converted_dict["ident"]) +'" />'   + "<div  class='heading'>" + converted_dict["fullname"] + '  <img style="float:right;margin-top:5px" src="' + survey_prefix + '/static/static_vo_tool/asc.gif"/></div>'
   7.330 +                            return_val += "<div id='item_description' style='display:none'> Description: "  + html_functions.escape(str(converted_dict["text"])) +'</div>'
   7.331 +                            return_val+='</div>'
   7.332 +                        elif type_helpers.isTable(_type):
   7.333 +                            return_val+='<div id="accordion_table">'
   7.334 +                         
   7.335 +                            converted_dict = dict([(str(k), v) for k, v in entry.items()])
   7.336 +                            return_val += '<input type="hidden" name="resource_type" value="' + converted_dict["type"]+ '" /> <input type="hidden" name="resource_ident" value="' + string_functions.encode(converted_dict["ident"]) +'" />'   + "<div  class='heading'>" + converted_dict["fullname"] + '  <img style="float:right;margin-top:5px" src="' + survey_prefix + '/static/static_vo_tool/asc.gif"/></div>'
   7.337 +                            return_val += "<div id='item_description'style='display:none'> Description: "  + html_functions.escape(str(converted_dict["text"])) + '<br /> Fullname: '  + str(converted_dict["fullname"]) +'</div>'
   7.338 +                            return_val+='</div>'
   7.339 +                       
   7.340 +                    
   7.341 +    except Exception:
   7.342 +        if f!="":
   7.343 +            f.close()    
   7.344 +        logging.exception("Error fetching metadata")
   7.345 +        return error_processing
   7.346 +    
   7.347 +    finally:
   7.348 +        if f!="":
   7.349 +            f.close()  
   7.350 +   
   7.351 +    return return_val
   7.352 +
   7.353 +def get_async_results(url, URI):
   7.354 +    """
   7.355 +    Open the given url and URI and read/return the result 
   7.356 +
   7.357 +    @param url: A URL string to open
   7.358 +    @param URI: A URI string to attach to the URL request
   7.359 +    @return: The result of the HTTP request sent to the URI of the URL
   7.360 +    """
   7.361 +    
   7.362 +    res = ''
   7.363 +    f = ''
   7.364 +    try:
   7.365 +        req = urllib2.Request(url+URI)
   7.366 +        f = urllib2.urlopen(req)
   7.367 +        res =  f.read()
   7.368 +        f.close()
   7.369 +    except Exception as e:
   7.370 +        if f!='':
   7.371 +            f.close()
   7.372 +        logging.exception('Exception caught:')
   7.373 +              
   7.374 +    return res
   7.375 +
   7.376 +
   7.377 +
   7.378 +def start_query_loop(url):
   7.379 +    """
   7.380 +    
   7.381 +    @param url: A URL string to be used
   7.382 +    @return: Results of query
   7.383 +    """
   7.384 +    
   7.385 +    def get_status(url):
   7.386 +        request2 = urllib2.Request(url, headers={"Accept" : "application/json", "firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : session.get("community_input","public (unknown)")})
   7.387 +        f_read = urllib2.urlopen(request2)
   7.388 +        query_json = f_read.read()
   7.389 +        return query_json
   7.390 +    
   7.391 +    max_size_exceeded = False 
   7.392 +   
   7.393 +    f_read = ""
   7.394 +    return_vot = ''
   7.395 +    delay = 2
   7.396 +    start_time = time.time()
   7.397 +    elapsed_time = 0
   7.398 +    query_json = {'syntax' : {'friendly' : 'A problem occurred while running your query', 'status' : 'Error' }}
   7.399 +    
   7.400 +    try:
   7.401 +     
   7.402 +        data = urllib.urlencode({ firethorn_config.query_status_update : "RUNNING"})
   7.403 +        request = urllib2.Request(url, data, headers={"Accept" : "application/json", "firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : session.get("community_input","public (unknown)")})
   7.404 +        f_update = urllib2.urlopen(request)
   7.405 +        query_json =  json.loads(f_update.read())
   7.406 +        query_status = query_json["status"]
   7.407 +        
   7.408 +        
   7.409 +        while query_status=="PENDING" or query_status=="RUNNING" and elapsed_time<3600:
   7.410 +            query_json = get_status(url)
   7.411 +            query_status= json.loads(query_json)["status"] 
   7.412 +            time.sleep(delay)
   7.413 +            elapsed_time = time.time() - start_time
   7.414 +       
   7.415 +        if query_status=="ERROR" or query_status=="FAILED":
   7.416 +            return {'Code' :-1,  'Content' : 'Query error: A problem occurred while running your query' }
   7.417 +        elif query_status=="CANCELLED":
   7.418 +            return {'Code' :1,  'Content' : 'Query error: Query has been canceled' }
   7.419 +        elif query_status=="EDITING":
   7.420 +            return {'Code' :-1,  'Content' : 'Query error: ' + query_json["syntax"]["status"] + ' - ' + query_json["syntax"]["friendly"] }
   7.421 +        elif query_status=="COMPLETED":
   7.422 +            return {'Code' :1,  'Content' : url + '/votable' }
   7.423 +        else :
   7.424 +            return {'Code' :-1,  'Content' : 'Query error: A problem occurred while running your query' }
   7.425 +        
   7.426 +    
   7.427 +    except Exception as e:
   7.428 +        logging.exception('Exception caught:')        
   7.429 +        return []
   7.430 +
   7.431 +    if f_read != "":
   7.432 +        f_read.close() 
   7.433 +    return return_vot    
   7.434 +
   7.435 +
   7.436 +
   7.437 +def start_async_loop(url):
   7.438 +    """
   7.439 +    Takes a TAP url and starts a loop that checks the phase URI and returns the results when completed. The loop is repeated every [delay=3] seconds
   7.440 +
   7.441 +    @param url: A URL string to be used
   7.442 +    @return: A Votable with the results of a TAP job, or '' if error
   7.443 +    """
   7.444 +    
   7.445 +    max_size_exceeded = False 
   7.446 +    f = ""
   7.447 +    return_vot = ''
   7.448 +    try:
   7.449 +        while True:
   7.450 +            res = get_async_results(url,'/phase')
   7.451 +            if res=='COMPLETED':       
   7.452 +                req = urllib2.Request(url + '/results/result')
   7.453 +                f = urllib2.urlopen(req)
   7.454 +                f.read(MAX_FILE_SIZE) #100Mb = 104,857,600
   7.455 +                if len(f.read())>0:
   7.456 +                    max_size_exceeded = True
   7.457 +                if not max_size_exceeded:
   7.458 +                    return_vot = atpy.Table(url + '/results/result', type='vo')
   7.459 +           
   7.460 +                else:
   7.461 +                    return_vot = "MAX_ERROR" 
   7.462 +                break
   7.463 +            elif res=='ERROR' or res== '':
   7.464 +                return return_vot
   7.465 +            time.sleep(delay)
   7.466 +    except Exception as e:
   7.467 +        logging.exception('Exception caught:')
   7.468 +        
   7.469 +        if f != "":
   7.470 +            f.close()
   7.471 +        return_vot = ''
   7.472 +    return return_vot    
   7.473 +
   7.474 +        
   7.475 +
   7.476 +
   7.477 +
   7.478 +
   7.479 +def print_save_HTML_table_info(pathname):
   7.480 +    """
   7.481 +    Generates HTML content for HTML table save button
   7.482 +        
   7.483 +    @param tbl: A table with the results of a query as a JSON, escaped array
   7.484 +    @return: The HTML content with a save as button
   7.485 +    """
   7.486 +        
   7.487 +
   7.488 +    result = '<div id="save_as"><form name="save_as_html" action="' + survey_sub_path + 'save_as_html" method="post"> <input type="hidden" name="pathname" value ="' 
   7.489 +    result += pathname + '"/><input type="submit" class ="save_button" value="Save as HTML" /></form></div>'
   7.490 +    return result
   7.491 +    
   7.492 +    
   7.493 +
   7.494 +
   7.495 +
   7.496 +
   7.497 +def print_save_VOTable_info(pathname):    
   7.498 +    """
   7.499 +    Generates HTML content for VOTable save button    
   7.500 +
   7.501 +    @param vot: A table with the results of a query as a JSON, escaped array
   7.502 +    @return: The HTML content with a save as button
   7.503 +    """
   7.504 +
   7.505 +    result = '<div id="save_as"><form name="save_as_vot" action="' + survey_sub_path + 'save_as_vot" method="post"> <input type="hidden" name="pathname" value ="' 
   7.506 +    result += pathname + '"/><input type="submit" class ="save_button" value="Save as Votable" /></form></div>'
   7.507 +    return result
   7.508 +
   7.509 +
   7.510 +
   7.511 +
   7.512 +
   7.513 +def print_save_Fits_info(pathname):    
   7.514 +    """
   7.515 +    Generates HTML content for VOTable save button    
   7.516 +
   7.517 +    @param vot: A table with the results of a query as a JSON, escaped array
   7.518 +    @return: The HTML content with a save as button
   7.519 +    """
   7.520 +    
   7.521 +    result = '<div id="save_as"><form name="save_as_fits" action="' + survey_sub_path + 'save_as_fits" method="post"> <input type="hidden" name="pathname" value ="' 
   7.522 +    result += pathname + '"/><input type="submit" class ="save_button" value="Save as Fits" /></form></div>'
   7.523 +    return result
   7.524 +
   7.525 +
   7.526 +
   7.527 +
   7.528 +def execute_async_query(url,mode_local,q,_format="votable"):
   7.529 +    """
   7.530 +    Execute an ADQL query (q) against a TAP service (url + mode:sync|async)       
   7.531 +    Starts by submitting a request for an async query, then uses the received job URL to call start_async_loop, to receive the final query results 
   7.532 +
   7.533 +    @param url: A string containing the TAP URL
   7.534 +    @param mode: sync or async to determine TAP mode of execution
   7.535 +    @param q: The ADQL Query to execute as string
   7.536 +    
   7.537 +    @return: Return a votable with the results, the TAP job ID and a temporary file path with the results stored on the server
   7.538 +    """    
   7.539 +
   7.540 +    votable = ''
   7.541 +    jobId= 'None'
   7.542 +    file_path=''
   7.543 +    rows=[]
   7.544 +    now = datetime.datetime.now()
   7.545 +    adql_table=''
   7.546 +    query_loop_results=''
   7.547 +    
   7.548 +    try:
   7.549 +
   7.550 +        # Return results as a votable object
   7.551 +       
   7.552 +        votable, adql_table, query_loop_results = run_query(q,"",query_space=url)
   7.553 +       
   7.554 +        if votable!='' and votable !="MAX_ERROR" and votable !="ERROR" and getattr(votable, "columns", None)!=None:
   7.555 +            
   7.556 +           
   7.557 +            clear_temp_folder()
   7.558 +            cols = list(votable.columns)
   7.559 +            
   7.560 +            lst = []
   7.561 +            
   7.562 +            rows = votable.data.tolist()
   7.563 +            row_length = len(rows)
   7.564 +            for x in rows:
   7.565 +                second_counter = 0
   7.566 +                obj_list = {}
   7.567 +                for y in x:
   7.568 +                  
   7.569 +                    obj_list[cols[second_counter]] = y 
   7.570 +                    second_counter = second_counter + 1
   7.571 +                lst.append(obj_list)
   7.572 +            lst = [cols,lst] 
   7.573 +            file_handle = File_handler()
   7.574 +            file_handle.create_temp_file()
   7.575 +            file_handle.write_to_temp_file(json.dumps(lst))
   7.576 +            file_handle.close_handle()    
   7.577 +            file_path = file_handle.pathname
   7.578 +            
   7.579 +    except Exception as e:
   7.580 +        logging.exception('Exception caught:')
   7.581 +
   7.582 + 
   7.583 +    try:
   7.584 +        if debug_mode != True:
   7.585 +            from web import ctx
   7.586 +           
   7.587 +            if votable=="" or votable==None or votable ==[] or getattr(votable, "columns", None)==None:
   7.588 +                userip =  "None" if str(web.ctx.ip)==None  else  str(web.ctx.ip)
   7.589 +                dbrelease = "Unknown" if url==None else url
   7.590 +                rows_error = -1
   7.591 +                MyOutputStream().log_queries(now.strftime('%Y-%m-%d %H:%M:%S.%f')[:-4], int((datetime.datetime.now() - now).total_seconds()), q, rows_error,rows_error, 
   7.592 +                                             html_functions.escape(session.get("username","unknown user")), html_functions.escape(dbrelease) , userip)
   7.593 +                
   7.594 +            else:
   7.595 +                
   7.596 +                userip =  "None" if str(web.ctx.ip)==None  else  str(web.ctx.ip)
   7.597 +                dbrelease = "Unknown" if url==None else url
   7.598 +    
   7.599 +                MyOutputStream().log_queries(now.strftime('%Y-%m-%d %H:%M:%S.%f')[:-4], int((datetime.datetime.now() - now).total_seconds()), q, len(rows),len(votable.columns), 
   7.600 +                                             html_functions.escape(session.get("username","unknown user")), html_functions.escape(dbrelease) , userip)
   7.601 +    except Exception:
   7.602 +        logging.exception('Exception caught:')
   7.603 +
   7.604 +
   7.605 +    return (votable,jobId,file_path,adql_table, query_loop_results)
   7.606 +
   7.607 +
   7.608 +
   7.609 +
   7.610 +def run_query(query=None, query_name="", query_space="", **kwargs):
   7.611 +    """
   7.612 +    Run a query on a resource
   7.613 +    """
   7.614 +    f=''
   7.615 +  
   7.616 +    query =  query
   7.617 +    query_space = string_functions.decode(query_space)
   7.618 +    result = ""
   7.619 +    max_size_exceeded = False
   7.620 +    result_adql_table = ""
   7.621 +    query_identity = ""
   7.622 +    
   7.623 +    try :
   7.624 +        from datetime import datetime
   7.625 +        t = datetime.now()
   7.626 +        if query_name=="":
   7.627 +            query_name = 'query-' + t.strftime("%y%m%d_%H%M%S")
   7.628 +     
   7.629 +        data = urllib.urlencode({ firethorn_config.query_name_param : query_name,  firethorn_config.query_param : query})
   7.630 +        request = urllib2.Request(query_space + firethorn_config.query_create_uri, data, headers={"Accept" : "application/json", "firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : session.get("community_input","public (unknown)")})
   7.631 +	
   7.632 +
   7.633 +	logging.exception(query)
   7.634 +
   7.635 +        f = urllib2.urlopen(request)
   7.636 +        query_create_result = json.loads(f.read())
   7.637 +        query_identity = query_create_result["ident"]
   7.638 +	logging.exception(query_identity)
   7.639 +
   7.640 +        query_loop_results = start_query_loop(query_identity)
   7.641 +        results_adql_url = query_create_result["results"]["adql"]
   7.642 +        
   7.643 +        try :
   7.644 +            req_rez_table = urllib2.Request( results_adql_url, headers={"Accept" : "application/json", "firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : session.get("community_input","public (unknown)")})
   7.645 +            response_rez_table = urllib2.urlopen(req_rez_table) 
   7.646 +            response_rez_table_json = response_rez_table.read()
   7.647 +            result_adql_table = json.loads(response_rez_table_json)["fullname"]
   7.648 +            response_rez_table.close()
   7.649 +        except Exception as e:
   7.650 +            logging.exception(e)
   7.651 +            
   7.652 +        if query_loop_results.get("Code", "") !="":
   7.653 +            if query_loop_results.get("Code", "") ==-1:
   7.654 +                result = query_loop_results.get("Content", "Error")
   7.655 +            elif query_loop_results.get("Code", "") ==1:
   7.656 +                req = urllib2.Request(query_loop_results.get("Content", ""), headers={"firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : session.get("community_input","public (unknown)")})
   7.657 +                f = urllib2.urlopen(req)
   7.658 +                f.read(MAX_FILE_SIZE) #100Mb = 104,857,600
   7.659 +             
   7.660 +                if len(f.read())>0:
   7.661 +                    max_size_exceeded = True
   7.662 +                if not max_size_exceeded:
   7.663 +                    result = atpy.Table(query_loop_results.get("Content", ""), type='vo')
   7.664 +                else:
   7.665 +                    result = ("MAX ERROR",result_adql_table,query_identity)
   7.666 +             
   7.667 +        else :
   7.668 +      
   7.669 +            return ("ERROR",result_adql_table,query_identity)
   7.670 +            
   7.671 +    except Exception:
   7.672 +        logging.exception('Exception caught:')
   7.673 +        
   7.674 +        return ("ERROR",result_adql_table,query_identity)
   7.675 +    
   7.676 +    if f!='':
   7.677 +        f.close()
   7.678 +    return (result,result_adql_table,query_identity)
   7.679 +    
   7.680 +    
   7.681 +
   7.682 +def generate_JSON_from_query((votable,jobId,pathname,adql_table,query_results_id),query="",tap_endpoint="",table_mode="static"):    
   7.683 +    """
   7.684 +    Generates HTML content for the results from an ADQL/TAP query    
   7.685 +    
   7.686 +    @param (votable,jobId,pathname): A tuple with the votable results, the jobID and the temporary file path
   7.687 +    @param query: The query that was executed
   7.688 +    @param tap_endpoint: The currently selected or generated TAP endpoint URL
   7.689 +    @return: The HTML content to be returned and asynchronously loaded on the users page after submitting a query
   7.690 +    """
   7.691 +    
   7.692 +    data_results = ""
   7.693 +    json_list = ""
   7.694 +    content = ""
   7.695 +
   7.696 +    if votable == "MAX_ERROR" or votable == "ERROR" or (getattr(votable, "columns", None)==None and (votable!='' and votable!= None)):
   7.697 +	data_results = ["ERROR"]	
   7.698 +	if votable == "MAX_ERROR":
   7.699 +	    data_results.append("Query exceeded maximum return size limit")
   7.700 +	elif votable == "ERROR":
   7.701 +  	    data_results.append("There was an error processing your request")
   7.702 +	else:	
   7.703 +	    data_results.append(votable)    
   7.704 +	if query_results_id!="" and query_results_id!=None:
   7.705 +	    query_id_content = '<div style="height:33px;text-align:left;margin-left:20px;font-size:13px;color:#C2BEAD">Query ID <a id="toggle_query_info">[+]</a><a id="toggle_query_info_help">[?]</a><div id="toggle_query_info_div" style="display:none">'+ query_results_id.split("/")[-1]  + '</div></div><br>' 
   7.706 +	else :
   7.707 +	    query_id_content = ""
   7.708 +	data_results.append(query_id_content)
   7.709 +	
   7.710 +        return json.dumps(data_results)
   7.711 +    elif votable!='' and votable!= None:
   7.712 +        if len(votable)>0:
   7.713 +            cols = list(votable.columns)
   7.714 +            row_list = votable.data.tolist() 
   7.715 +            if tap_endpoint!="" and query!="":
   7.716 +                content = '<form class="launch_viewer" style="float:left;z-index:100;position: relative;" action="' + survey_prefix + '/viewer" method="post" target="_blank"><input type="hidden" name="query" value="'+ string_functions.encode(query) +'"/><input type="hidden" name="cols" value="'+ html_functions.escape(json.dumps(cols)) +'"/><input type="hidden" name="filepath" id="temp_file" value="'+ pathname +'"/><input type="hidden" name="tap_endpoint" value="'+ tap_endpoint +'"/>'
   7.717 +                content += '<input type="hidden" name="adql_table" id="adql_table" value="'+ adql_table +'"/>'
   7.718 +                if table_mode == 'interactive':
   7.719 +                    data_results = html_functions.escape_list(row_list)
   7.720 +                    json_list = html_functions.escape(json.dumps(data_results))
   7.721 +                    content += '<input type="hidden" id="hidden_json_results" name="results" value="' + json_list  + '" />'
   7.722 +                content += '<div style="margin-bottom:5px;text-align:left;margin-left:20px;font-size:13px;color:#C2BEAD">Launch in Plotter<input type="submit" value="" /></form></div><br /><br />'
   7.723 +                content += '<div style="height:33px;clear:both;text-align:left;margin-left:20px;font-size:13px;color:#C2BEAD">Query ID <a id="toggle_query_info">[+]</a><a id="toggle_query_info_help">[?]</a><div id="toggle_query_info_div" style="display:none">'+ query_results_id.split("/")[-1]  + '</div></div><br>'
   7.724 +
   7.725 +            else:
   7.726 +                content = '<input type="hidden" name="filepath" id="temp_file" value="'+ pathname +'"/>'
   7.727 +            content += '<div id="save_as_information" style="display:none">' + print_save_VOTable_info(pathname) + print_save_HTML_table_info(pathname) + print_save_Fits_info(pathname) + '</div>'
   7.728 +            content += '<div style="text-align:left;margin-left:15px;float:left;z-index:100;position: relative;clear:both"><img id ="samp_connection" src="' + survey_prefix + '/static/images/red-button.png" height="17" width="17" /><button id="register">Connect to SAMP</button><button id="unregister" style="display: none;">Disconnect from SAMP</button><button id="loadvotable" style="display: none;">Broadcast results table</button></div>'
   7.729 +
   7.730 +            data_results = [cols]
   7.731 +            data_results.append([len(row_list)])
   7.732 +            data_results.append([content])
   7.733 +            return json.dumps(data_results)
   7.734 +        else:
   7.735 +            return json.dumps([])
   7.736 +    else:
   7.737 +
   7.738 +           
   7.739 +	data_results = ["ERROR"]
   7.740 +	if votable == "MAX_ERROR":
   7.741 +	    data_results.append("Query exceeded maximum return size limit")
   7.742 +	elif votable == "ERROR":
   7.743 +  	    data_results.append("There was an error processing your request")
   7.744 +	else:	
   7.745 +	    data_results.append(votable)    
   7.746 +
   7.747 +	if query_results_id!="" and query_results_id!=None:
   7.748 +	    query_id_content = '<div style="height:33px;text-align:left;margin-left:20px;font-size:13px;color:#C2BEAD">Query ID <a id="toggle_query_info">[+]</a><a id="toggle_query_info_help">[?]</a><div id="toggle_query_info_div" style="display:none"> '+ query_results_id.split("/")[-1]  + '</div></div><br>' 
   7.749 +	else :
   7.750 +	    query_id_content = ""
   7.751 +
   7.752 +	data_results.append(query_id_content)     
   7.753 +	return data_results
   7.754 +
   7.755 +
   7.756 +
   7.757 +def execute_async_region_query(url,mode_local,q):
   7.758 +    """
   7.759 +    Execute an ADQL query (q) against a TAP service (url + mode:sync|async)       
   7.760 +    Starts by submitting a request for an async query, then uses the received job URL to call start_async_loop, to receive the final query results 
   7.761 +
   7.762 +    @param url: A string containing the TAP URL
   7.763 +    @param mode: sync or async to determine TAP mode of execution
   7.764 +    @param q: The ADQL Query to execute as string
   7.765 +    
   7.766 +    @return: Return a votable with the results, the TAP job ID and a temporary file path with the results stored on the server
   7.767 +    """    
   7.768 +
   7.769 +    params = urllib.urlencode({'REQUEST': request, 'LANG': lang, 'FORMAT': result_format, 'QUERY' : q}) 
   7.770 +    full_url = url+mode_local
   7.771 +
   7.772 +    votable = ''
   7.773 +    jobId= 'None'
   7.774 +    file_path=''
   7.775 +    try:
   7.776 +        
   7.777 +        #Submit job and get job id 
   7.778 +        req = urllib2.Request(full_url, params)
   7.779 +        opener = urllib2.build_opener()
   7.780 +  
   7.781 +        f = opener.open(req)
   7.782 +        jobId = f.url
   7.783 +  
   7.784 +        #Execute job and start loop requests for results
   7.785 +        req2 = urllib2.Request(jobId+'/phase',urllib.urlencode({'PHASE' : 'RUN'}))
   7.786 +        f2 = opener.open(req2) #@UnusedVariable
   7.787 +
   7.788 +        # Return results as a votable object
   7.789 +        votable = start_async_loop(jobId)
   7.790 +        
   7.791 +        if votable!='' and votable !="MAX_ERROR":
   7.792 +            clear_temp_folder()
   7.793 +            cols = list(votable.columns)
   7.794 +            cols.insert(0,"getFSLink")
   7.795 +            col_data_to_list =  list(votable.columns)
   7.796 +            
   7.797 +            if 'ra' in col_data_to_list:
   7.798 +                ra_index = col_data_to_list.index('ra')
   7.799 +            else :
   7.800 +                ra_index = -1
   7.801 +                
   7.802 +            if 'dec' in col_data_to_list:
   7.803 +                dec_index = col_data_to_list.index('dec')
   7.804 +            else :
   7.805 +                dec_index = -1
   7.806 +                
   7.807 +            if  'frameSetID' in col_data_to_list:
   7.808 +                framesetID_index = col_data_to_list.index('frameSetID')
   7.809 +            else:
   7.810 +                framesetID_index = -1
   7.811 +                if 'multiframeID' in col_data_to_list:
   7.812 +                    multiframeID_index = col_data_to_list.index('multiframeID')
   7.813 +                else:
   7.814 +                    multiframeID_index = -1
   7.815 +
   7.816 +
   7.817 +            db = SURVEY_DB
   7.818 +           
   7.819 +            temp_row=[]
   7.820 +            for i in votable.data.tolist():
   7.821 +                if framesetID_index == -1 and multiframeID_index == -1:
   7.822 +                    getFSLink = ''
   7.823 +                
   7.824 +                else :
   7.825 +                    getFSLink = '<a target="getI" href="' + getImageURL +'?mode=show&ra=' + str(i[ra_index])  + '&dec=' + str(i[dec_index]) + '&fsid=' + str(i[framesetID_index]) + '&database=' + db + '&archive=' + archive_input +'">view</a>'
   7.826 +                temp_row.append(getFSLink)
   7.827 +           
   7.828 +            votable.add_column(name="getFSLink",data=temp_row,position=0)
   7.829 +            
   7.830 +            
   7.831 +            lst = []
   7.832 +            for x in votable.data.tolist():
   7.833 +                second_counter = 0
   7.834 +                obj_list = {}
   7.835 +                for y in x:
   7.836 +                    obj_list[cols[second_counter]] = y 
   7.837 +                    second_counter = second_counter + 1
   7.838 +                lst.append(obj_list)
   7.839 +            
   7.840 +            #html_functions.escape_list(votable.data.tolist())
   7.841 +            lst = [cols,lst] 
   7.842 +            file_handle = File_handler()
   7.843 +            file_handle.create_temp_file()
   7.844 +            file_handle.write_to_temp_file(json.dumps(lst))
   7.845 +            file_handle.close_handle()    
   7.846 +            file_path = file_handle.pathname
   7.847 +            
   7.848 +    except Exception as e:
   7.849 +        logging.exception('Exception caught:')
   7.850 +
   7.851 +    return (votable,jobId,file_path,"",jobId)
   7.852 +
   7.853 +
   7.854 +
   7.855 +def execute_async_imagelist_query(cols,rows):
   7.856 +    """
   7.857 +    Generate results for imagelist query
   7.858 +    """    
   7.859 +  
   7.860 +  
   7.861 +    votable = ''
   7.862 +    file_path=''
   7.863 +
   7.864 +    try:
   7.865 +        
   7.866 +        if cols!=None and rows !=None:
   7.867 +            clear_temp_folder()
   7.868 + 
   7.869 +            col_data_to_list =  cols
   7.870 +            votable = atpy.Table()
   7.871 +            cntr = 0
   7.872 +            
   7.873 +            for i in cols:
   7.874 +                temp_row = []
   7.875 +            
   7.876 +                for y in rows:
   7.877 +                    if type(y[cntr]) == long:
   7.878 +                        temp_row.append(float(y[cntr]))
   7.879 +                    elif type(y[cntr]) == int or type(y[cntr]) == float:
   7.880 +                        temp_row.append(y[cntr])
   7.881 +                    else:
   7.882 +                        temp_row.append(str(y[cntr]))
   7.883 +                   
   7.884 +                votable.add_column(i,data=temp_row,position=cntr)
   7.885 +                cntr = cntr + 1
   7.886 +                
   7.887 +            filename = col_data_to_list.index('filename')
   7.888 +            catname = col_data_to_list.index('catname')
   7.889 +            jpegBase = col_data_to_list.index('jpegBase')
   7.890 +            ext = col_data_to_list.index('numDetectors')
   7.891 +            mfid = col_data_to_list.index('multiframeID')
   7.892 +            rID = 1
   7.893 +
   7.894 +            temp_row_view=[]
   7.895 +            temp_row_img=[]
   7.896 +            temp_row_cat=[]
   7.897 +           
   7.898 +            for i in votable.data.tolist():
   7.899 +
   7.900 +                view = '<a id="ImageList_facebox" target="display" href="' + survey_cgi_bin + 'display.cgi?file=' + str(i[filename]) + '&cat=' + str(i[catname]) + '&comp=' + str(i[jpegBase]) + '&noExt=' + str(i[ext]) + '&MFID=' + str(i[mfid]) + '&rID=' + str(rID) +'">view</a>'
   7.901 +                img = '<a  class="ImageList_dllink" href="' + survey_cgi_bin + 'fits_download.cgi?file=' + str(i[filename]) +'&MFID=' + str(i[mfid]) + '&rID=' + str(rID) +'">FITS</a>'
   7.902 +                cat = '<a  class="ImageList_dllink" href="' + survey_cgi_bin + 'fits_download.cgi?file=' + str(i[catname]) + '&amp;MFID=' + str(i[mfid]) + '&amp;rID=' + str(rID) + '">FITS</a>'
   7.903 +                temp_row_view.append(view)
   7.904 +                temp_row_img.append(img)
   7.905 +                temp_row_cat.append(cat)
   7.906 +            
   7.907 +            votable.remove_columns(['jpegBase','filename','catname'])
   7.908 +
   7.909 +            votable.add_column(name="View",data=temp_row_view,position=0)
   7.910 +            votable.add_column(name="Img",data=temp_row_img,position=1)
   7.911 +            votable.add_column(name="Cat",data=temp_row_cat,position=2)
   7.912 +
   7.913 +            cols = list(votable.columns)
   7.914 +            
   7.915 +            lst = []
   7.916 +            for x in votable.data.tolist():
   7.917 +                second_counter = 0
   7.918 +                obj_list = {}
   7.919 +                for y in x:
   7.920 +                    obj_list[cols[second_counter]] = y 
   7.921 +                    second_counter = second_counter + 1
   7.922 +                lst.append(obj_list)
   7.923 +            
   7.924 +            #html_functions.escape_list(votable.data.tolist())
   7.925 +            final_lst = [cols,lst] 
   7.926 +            file_handle = File_handler()
   7.927 +            file_handle.create_temp_file()
   7.928 +            file_handle.write_to_temp_file(json.dumps(final_lst))
   7.929 +            file_handle.close_handle()    
   7.930 +            file_path = file_handle.pathname
   7.931 +
   7.932 +           
   7.933 +    except Exception as e:
   7.934 +        logging.exception('Exception caught:')
   7.935 +    return (votable,"",file_path,"","")
   7.936 +   
   7.937 +
   7.938 + 
   7.939 +def write_format_from_json(to_format,json_list, visible_cols=None):
   7.940 +    """
   7.941 +    Generates HTML to_format for a json array of data
   7.942 +    
   7.943 +    @param to_format: The format to which to transform a JSON list of data
   7.944 +    @param json_list: The source json list
   7.945 +    """
   7.946 +
   7.947 +    html_array = json_list
   7.948 +    fields = html_array[0] if visible_cols == None else visible_cols
   7.949 +    data = html_array[1]
   7.950 +    t = atpy.Table()
   7.951 +    
   7.952 +    if to_format == 'csv':
   7.953 +   
   7.954 +        csv = ''
   7.955 +        counter = 0 
   7.956 +        if visible_cols != None:
   7.957 +            for col in fields:
   7.958 +                counter = counter + 1
   7.959 +                if counter == len(fields):
   7.960 +                    csv += str(col)
   7.961 +                else:
   7.962 +                    csv += str(col) + ','
   7.963 +       
   7.964 +        csv += '\n'
   7.965 +       
   7.966 +        for row in data:
   7.967 +            values = []
   7.968 +            
   7.969 +            for col in fields:
   7.970 +	   	try:   	
   7.971 +	            values.append(str(row[col]))
   7.972 +           	except Exception as e:
   7.973 +		    logging.exception(e)
   7.974 +		
   7.975 +            csv += ",".join(values) 
   7.976 +            csv += '\n'
   7.977 +            
   7.978 +        result = csv
   7.979 +    
   7.980 +    else:    
   7.981 +        
   7.982 +        for val in fields:
   7.983 +            
   7.984 +            col_data = []
   7.985 +            for row in data:
   7.986 +             
   7.987 +                if(type(row[val])==unicode):
   7.988 +                    col_data.append(numpy.str(row[val]))
   7.989 +                    mytype = numpy.string_
   7.990 +                else:
   7.991 +                    col_data.append(row[val])
   7.992 +                    mytype = type(row[val])
   7.993 +            try:
   7.994 +                t.add_column(str(val),numpy.array(col_data),dtype=mytype)
   7.995 +            except Exception as e:
   7.996 +                t.add_column(str(val),numpy.array(col_data),dtype=numpy.string_)
   7.997 +    
   7.998 +        d = StringIO.StringIO()
   7.999 +        t.write(d, type=to_format)
  7.1000 +        result = d.getvalue()
  7.1001 +        d.close()
  7.1002 +
  7.1003 +    
  7.1004 +    return result            
  7.1005 +
  7.1006 +
  7.1007 +
  7.1008 +def generate_query_add_secondary_order(query, colid):
  7.1009 +    """
  7.1010 +    Add a secondary order clause to an sql query to help with data consistency when data from query returned has the same value
  7.1011 +    """
  7.1012 +    query_as_string = query
  7.1013 +    if query.lower().find(" order ")>0:
  7.1014 +        query = query.replace('(', ' ( ')
  7.1015 +        query = query.replace(')', ' ) ')
  7.1016 +        query = query.replace(',', ' , ')
  7.1017 +        query_list = query.split(" ")
  7.1018 +        query_list = filter(None, query_list)
  7.1019 +        new_query_list = []
  7.1020 +        prev_item = ""
  7.1021 +        prev_prev_item = ""
  7.1022 +        secondary_order = "," + colid + " "
  7.1023 +        is_order_expression = False
  7.1024 +        insert_next = False
  7.1025 +
  7.1026 +        for item in query_list:
  7.1027 +            if prev_item.lower() == "by" and prev_prev_item.lower() == "order":
  7.1028 +                is_order_expression = True
  7.1029 +            
  7.1030 +            if is_order_expression == True:
  7.1031 +                if insert_next == True and item!="," and item.find(',')<0: 
  7.1032 +                    new_query_list.append(secondary_order)
  7.1033 +                    new_query_list.append(item + " ")
  7.1034 +                    insert_next=False
  7.1035 +                    is_order_expression = False
  7.1036 +                elif item!="," and item!="" and item.find(',')<0 :
  7.1037 +                    insert_next = True 
  7.1038 +                    new_query_list.append(item + " ")
  7.1039 +                else :
  7.1040 +                    insert_next=False
  7.1041 +                    new_query_list.append(item + " ")
  7.1042 +                 
  7.1043 +            else:
  7.1044 +                new_query_list.append(item + " ")
  7.1045 +            
  7.1046 +          
  7.1047 +                                                
  7.1048 +            prev_prev_item = prev_item
  7.1049 +            prev_item = item
  7.1050 +        
  7.1051 +        if is_order_expression == True:
  7.1052 +            new_query_list.append(secondary_order)
  7.1053 +
  7.1054 +        query_as_string = "".join(new_query_list)
  7.1055 +    
  7.1056 +    else :
  7.1057 +        query_as_string = query 
  7.1058 +        query_as_string += " order by " + colid 
  7.1059 +    return query_as_string
  7.1060 +    
  7.1061 +    
  7.1062 +def get_parent_workspace(schema):
  7.1063 +    """
  7.1064 +    Get the parent workspace of a schema
  7.1065 +    """
  7.1066 +    
  7.1067 +    req = urllib2.Request( schema, headers={"Accept" : "application/json", "firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : session.get("community_input","public (unknown)")} )
  7.1068 +    response = urllib2.urlopen(req)
  7.1069 +    workspace = json.loads(response.read())["parent"]
  7.1070 +    response.close()
  7.1071 +    return workspace
  7.1072 +
  7.1073 +
  7.1074 +def get_info(obj):
  7.1075 +    """
  7.1076 +    Get the parent workspace of a schema
  7.1077 +    """
  7.1078 +    
  7.1079 +    req = urllib2.Request( obj, headers={"Accept" : "application/json", "firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : session.get("community_input","public (unknown)")} )
  7.1080 +    response = urllib2.urlopen(req)
  7.1081 +    info = json.loads(response.read())
  7.1082 +    response.close()
  7.1083 +    return info
     8.1 --- a/src/url_classes/viewer.py	Sun Feb 09 23:42:28 2014 +0000
     8.2 +++ b/src/url_classes/viewer.py	Sun Feb 09 23:48:43 2014 +0000
     8.3 @@ -54,7 +54,7 @@
     8.4          adql_table = string_functions.decode(data.adql_table)
     8.5          paren_open = ""
     8.6          paren_close = ""
     8.7 -        
     8.8 +	
     8.9          if adql_table!="" and adql_table!=None:
    8.10              query = adql_table
    8.11  
    8.12 @@ -76,19 +76,11 @@
    8.13                      _format = 'votable/td'
    8.14  
    8.15              votable,jobid, plot_file,adql_table, query_results_id = freeform_sql.execute_async_query(data.tap_endpoint,mode_global,new_query,_format) #@UnusedVariable
    8.16 -            json_data = json.loads(votable)
    8.17 -            if len(json_data)<2:
    8.18 -                votable = ""
    8.19 -            else :
    8.20 -                cols = json_data[0]
    8.21 -                row_length = len(json_data[1])
    8.22 -                rows = []
    8.23 -                for x in json_data[1]:
    8.24 -                    rows.append(x.values())
    8.25 +       
    8.26              if votable!='':            
    8.27 -                if row_length>0:
    8.28 +                if len(votable)>0:
    8.29                      try:
    8.30 -                        data_to_list = rows     
    8.31 +                        data_to_list = votable.data.tolist()     
    8.32                          if (data.plot_type=="Scatter" or data.plot_type == "Density" or data.plot_type == "Mixed"):
    8.33                              if (type (data_to_list[0][0]) is str) or (type (data_to_list[0][1]) is str):
    8.34                                  data_to_list = [map (self.tryeval,x) for x in data_to_list]
    8.35 @@ -100,7 +92,7 @@
    8.36                                  if (type (data_to_list[0][0]) is str):
    8.37                                      raise Exception
    8.38                                  
    8.39 -                        data_results = [cols]
    8.40 +                        data_results = [list(votable.columns)]
    8.41                          data_results.append(data_to_list)
    8.42                          
    8.43                          if data.mode=="static":    
    8.44 @@ -117,16 +109,17 @@
    8.45                          
    8.46                      except Exception as e:
    8.47                          logging.exception('Exception caught:')
    8.48 -                        return_value = "ERROR"   
    8.49 +                        return_value = "<div style=\"color:red;margin-left:50px;font-size:12px\">There was an error processing your request</div>"   
    8.50              else : 
    8.51 -                return_value = "ERROR"    
    8.52 +                return_value = "<div style=\"color:red;margin-left:50px;font-size:12px\">There was an error processing your request</div>"    
    8.53          table_cols = data.cols
    8.54          
    8.55          if table_cols == None or table_cols == "":
    8.56              table_cols = 0
    8.57 +        
    8.58 +        if return_value=='':
    8.59         
    8.60 -        if return_value=='':
    8.61 -            return render.viewer(table_cols,'value="'+ query + '"','value="'+ data.tap_endpoint + '"','value="'+ data.filepath + '"', survey_prefix, rand)   
    8.62 +            return render.viewer(table_cols,'value="'+ query + '"','value="'+ data.tap_endpoint + '"','value="'+ data.filepath + '"', 'value="' + freeform_sql.html_functions.escape(data.results) +'"', survey_prefix)   
    8.63          else :
    8.64              return return_value
    8.65