clearwing

changeset 12:57d7ec9bbf7f 1.0.0

Replace atpy VOTable use with java custom datatables controller
author Stelios Voutsinas <stv@roe.ac.uk>
date Thu Jan 30 20:43:55 2014 +0000 (2014-01-30)
parents afc80802e308
children 4df973a046de
files src/freeform_sql/misc.py src/freeform_sql/misc.pyc src/static/js/properties.js src/static/static_crossID_form/js/crossID.js src/templates/dbaccess_login.html src/templates/viewer.html src/url_classes/data_tables_processing.pyc src/url_classes/dbaccess_SQL_form.py src/url_classes/dbaccess_SQL_form.pyc src/url_classes/helper_urls.py src/url_classes/helper_urls.pyc src/url_classes/viewer.py src/url_classes/viewer.pyc
line diff
     1.1 --- a/src/freeform_sql/misc.py	Tue Jan 21 17:52:13 2014 +0000
     1.2 +++ b/src/freeform_sql/misc.py	Thu Jan 30 20:43:55 2014 +0000
     1.3 @@ -23,13 +23,13 @@
     1.4  from file_handler import File_handler
     1.5  from helper_functions.string_functions import string_functions
     1.6  string_functions = string_functions()
     1.7 -from config import survey_prefix,SURVEY_DB,base_location, debug_mode
     1.8 +from config import survey_prefix, SURVEY_DB,base_location, debug_mode, MAX_DELAY, MAX_ELAPSED_TIME, MIN_ELAPSED_TIME_BEFORE_REDUCE, INITIAL_DELAY
     1.9  from survey_globals import archive_input
    1.10  from globals import MyOutputStream, logging
    1.11  import datetime
    1.12  import firethorn_config
    1.13  from app import session
    1.14 -
    1.15 +from time import gmtime,  strftime
    1.16  
    1.17  def clear_temp_folder():
    1.18      """
    1.19 @@ -389,26 +389,35 @@
    1.20     
    1.21      f_read = ""
    1.22      return_vot = ''
    1.23 -    delay = 2
    1.24 -    start_time = time.time()
    1.25 +    delay = INITIAL_DELAY
    1.26 +    start_time = datetime.datetime.now()
    1.27      elapsed_time = 0
    1.28      query_json = {'syntax' : {'friendly' : 'A problem occurred while running your query', 'status' : 'Error' }}
    1.29      
    1.30      try:
    1.31 -     
    1.32 +        logging.exception("Started Firethorn job :::" +  strftime("%Y-%m-%d %H:%M:%S", gmtime()))
    1.33 +
    1.34          data = urllib.urlencode({ firethorn_config.query_status_update : "RUNNING"})
    1.35          request = urllib2.Request(url, data, headers={"Accept" : "application/json", "firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : session.get("community_input","public (unknown)")})
    1.36          f_update = urllib2.urlopen(request)
    1.37          query_json =  json.loads(f_update.read())
    1.38          query_status = query_json["status"]
    1.39          
    1.40 -        
    1.41 -        while query_status=="PENDING" or query_status=="RUNNING" and elapsed_time<3600:
    1.42 +        logging.exception("delay_initial: " + str(delay))
    1.43 +
    1.44 +        while query_status=="PENDING" or query_status=="RUNNING" and elapsed_time<MAX_ELAPSED_TIME:
    1.45              query_json = get_status(url)
    1.46              query_status= json.loads(query_json)["status"] 
    1.47              time.sleep(delay)
    1.48 -            elapsed_time = time.time() - start_time
    1.49 -       
    1.50 +            if elapsed_time>MIN_ELAPSED_TIME_BEFORE_REDUCE and delay<MAX_DELAY:
    1.51 +                delay = delay + delay
    1.52 +            elapsed_time = (datetime.datetime.now() - start_time).seconds
    1.53 +            logging.exception("elapsed_time: ")
    1.54 +            logging.exception(elapsed_time)
    1.55 +            logging.exception("delay: " + str(delay))
    1.56 +            
    1.57 +        logging.exception("Completed Firethorn job :::" +  strftime("%Y-%m-%d %H:%M:%S", gmtime()))
    1.58 +  
    1.59          if query_status=="ERROR" or query_status=="FAILED":
    1.60              return {'Code' :-1,  'Content' : 'Query error: A problem occurred while running your query' }
    1.61          elif query_status=="CANCELLED":
    1.62 @@ -416,7 +425,7 @@
    1.63          elif query_status=="EDITING":
    1.64              return {'Code' :-1,  'Content' : 'Query error: ' + query_json["syntax"]["status"] + ' - ' + query_json["syntax"]["friendly"] }
    1.65          elif query_status=="COMPLETED":
    1.66 -            return {'Code' :1,  'Content' : url + '/votable' }
    1.67 +            return {'Code' :1,  'Content' : url + '/datatable' }
    1.68          else :
    1.69              return {'Code' :-1,  'Content' : 'Query error: A problem occurred while running your query' }
    1.70          
    1.71 @@ -541,23 +550,25 @@
    1.72      now = datetime.datetime.now()
    1.73      adql_table=''
    1.74      query_loop_results=''
    1.75 +    cols = []
    1.76 +    rows = []
    1.77      
    1.78      try:
    1.79  
    1.80          # Return results as a votable object
    1.81 -       
    1.82 +        logging.exception("Started run_query ::::::" +  strftime("%Y-%m-%d %H:%M:%S", gmtime()))
    1.83          votable, adql_table, query_loop_results = run_query(q,"",query_space=url)
    1.84 -       
    1.85 -        if votable!='' and votable !="MAX_ERROR" and votable !="ERROR" and getattr(votable, "columns", None)!=None:
    1.86 +        if votable!='' and votable !="MAX_ERROR" and votable !="ERROR" and votable!=None:
    1.87              
    1.88             
    1.89              clear_temp_folder()
    1.90 -            cols = list(votable.columns)
    1.91 -            
    1.92 +           
    1.93 +            #cols = list(votable.columns)
    1.94 +            '''
    1.95              lst = []
    1.96              
    1.97 -            rows = votable.data.tolist()
    1.98 -            row_length = len(rows)
    1.99 +            #rows = votable.data.tolist()
   1.100 +            #row_length = len(rows)
   1.101              
   1.102              
   1.103              for x in rows:
   1.104 @@ -569,9 +580,11 @@
   1.105                      second_counter = second_counter + 1
   1.106                  lst.append(obj_list)
   1.107              lst = [cols,lst] 
   1.108 +            '''
   1.109 +            
   1.110              file_handle = File_handler()
   1.111              file_handle.create_temp_file()
   1.112 -            file_handle.write_to_temp_file(json.dumps(lst))
   1.113 +            file_handle.write_to_temp_file(votable)
   1.114              file_handle.close_handle()    
   1.115              file_path = file_handle.pathname
   1.116              
   1.117 @@ -583,7 +596,7 @@
   1.118          if debug_mode != True:
   1.119              from web import ctx
   1.120             
   1.121 -            if votable=="" or votable==None or votable ==[] or getattr(votable, "columns", None)==None:
   1.122 +            if votable=="" or votable==None or votable ==[] :
   1.123                  userip =  "None" if str(web.ctx.ip)==None  else  str(web.ctx.ip)
   1.124                  dbrelease = "Unknown" if url==None else url
   1.125                  rows_error = -1
   1.126 @@ -595,11 +608,12 @@
   1.127                  userip =  "None" if str(web.ctx.ip)==None  else  str(web.ctx.ip)
   1.128                  dbrelease = "Unknown" if url==None else url
   1.129      
   1.130 -                MyOutputStream().log_queries(now.strftime('%Y-%m-%d %H:%M:%S.%f')[:-4], int((datetime.datetime.now() - now).total_seconds()), q, len(rows),len(votable.columns), 
   1.131 +                MyOutputStream().log_queries(now.strftime('%Y-%m-%d %H:%M:%S.%f')[:-4], int((datetime.datetime.now() - now).total_seconds()), q, len(rows),len(cols), 
   1.132                                               html_functions.escape(session.get("username","unknown user")), html_functions.escape(dbrelease) , userip)
   1.133      except Exception:
   1.134          logging.exception('Exception caught:')
   1.135  
   1.136 +    logging.exception("Completed run_query :::" +  strftime("%Y-%m-%d %H:%M:%S", gmtime()))
   1.137  
   1.138      return (votable,jobId,file_path,adql_table, query_loop_results)
   1.139  
   1.140 @@ -632,7 +646,6 @@
   1.141          f = urllib2.urlopen(request)
   1.142          query_create_result = json.loads(f.read())
   1.143          query_identity = query_create_result["ident"]
   1.144 -
   1.145          query_loop_results = start_query_loop(query_identity)
   1.146          results_adql_url = query_create_result["results"]["adql"]
   1.147          
   1.148 @@ -651,12 +664,12 @@
   1.149              elif query_loop_results.get("Code", "") ==1:
   1.150                  req = urllib2.Request(query_loop_results.get("Content", ""), headers={"firethorn.auth.identity" : session.get("email","unknown user"), "firethorn.auth.community" : session.get("community_input","public (unknown)")})
   1.151                  f = urllib2.urlopen(req)
   1.152 -                f.read(MAX_FILE_SIZE) #100Mb = 104,857,600
   1.153 +                datatable = f.read(MAX_FILE_SIZE) 
   1.154               
   1.155                  if len(f.read())>0:
   1.156                      max_size_exceeded = True
   1.157                  if not max_size_exceeded:
   1.158 -                    result = atpy.Table(query_loop_results.get("Content", ""), type='vo')
   1.159 +                    result = datatable
   1.160                  else:
   1.161                      result = ("MAX ERROR",result_adql_table,query_identity)
   1.162               
   1.163 @@ -668,7 +681,6 @@
   1.164          logging.exception('Exception caught:')
   1.165          
   1.166          return ("ERROR",result_adql_table,query_identity)
   1.167 -    
   1.168      if f!='':
   1.169          f.close()
   1.170      return (result,result_adql_table,query_identity)
   1.171 @@ -686,35 +698,47 @@
   1.172      """
   1.173      
   1.174      data_results = ""
   1.175 -    json_list = ""
   1.176      content = ""
   1.177 +    json_data = json.loads(votable)
   1.178 +    
   1.179 +    if len(json_data)<2:
   1.180 +        votable = "ERROR"
   1.181 +    else :
   1.182 +        cols = json_data[0]
   1.183 +        row_length = len(json_data[1])
   1.184 +    
   1.185 +    
   1.186 +    logging.exception("Starting generate_JSON_from_query :::" +  strftime("%Y-%m-%d %H:%M:%S", gmtime()))
   1.187  
   1.188 -    if votable == "MAX_ERROR" or votable == "ERROR" or (getattr(votable, "columns", None)==None and (votable!='' and votable!= None)):
   1.189 -	data_results = ["ERROR"]	
   1.190 -	if votable == "MAX_ERROR":
   1.191 -	    data_results.append("Query exceeded maximum return size limit")
   1.192 -	elif votable == "ERROR":
   1.193 -  	    data_results.append("There was an error processing your request")
   1.194 -	else:	
   1.195 -	    data_results.append(votable)    
   1.196 -	if query_results_id!="" and query_results_id!=None:
   1.197 -	    query_id_content = '<div style="height:33px;text-align:left;margin-left:20px;font-size:13px;color:#C2BEAD">Query ID <a id="toggle_query_info">[+]</a><a id="toggle_query_info_help">[?]</a><div id="toggle_query_info_div" style="display:none">'+ query_results_id.split("/")[-1]  + '</div></div><br>' 
   1.198 -	else :
   1.199 -	    query_id_content = ""
   1.200 -	data_results.append(query_id_content)
   1.201 -	
   1.202 +    if votable == "MAX_ERROR" or votable == "ERROR":
   1.203 +        data_results = ["ERROR"]	
   1.204 +        if votable == "MAX_ERROR":
   1.205 +            data_results.append("Query exceeded maximum return size limit")
   1.206 +        elif votable == "ERROR":
   1.207 +            data_results.append("There was an error processing your request")
   1.208 +        else:	
   1.209 +            data_results.append(votable)    
   1.210 +    
   1.211 +    
   1.212 +        if query_results_id!="" and query_results_id!=None:
   1.213 +            query_id_content = '<div style="height:33px;text-align:left;margin-left:20px;font-size:13px;color:#C2BEAD">Query ID <a id="toggle_query_info">[+]</a><a id="toggle_query_info_help">[?]</a><div id="toggle_query_info_div" style="display:none">'+ query_results_id.split("/")[-1]  + '</div></div><br>' 
   1.214 +        else :
   1.215 +            query_id_content = ""
   1.216 +        
   1.217 +        data_results.append(query_id_content)
   1.218 +        
   1.219          return json.dumps(data_results)
   1.220 +    
   1.221      elif votable!='' and votable!= None:
   1.222 -        if len(votable)>0:
   1.223 -            cols = list(votable.columns)
   1.224 -            row_list = votable.data.tolist() 
   1.225 +        if row_length>0:
   1.226 +            cols = list(cols)
   1.227              if tap_endpoint!="" and query!="":
   1.228                  content = '<form class="launch_viewer" style="float:left;z-index:100;position: relative;" action="' + survey_prefix + '/viewer" method="post" target="_blank"><input type="hidden" name="query" value="'+ string_functions.encode(query) +'"/><input type="hidden" name="cols" value="'+ html_functions.escape(json.dumps(cols)) +'"/><input type="hidden" name="filepath" id="temp_file" value="'+ pathname +'"/><input type="hidden" name="tap_endpoint" value="'+ tap_endpoint +'"/>'
   1.229                  content += '<input type="hidden" name="adql_table" id="adql_table" value="'+ adql_table +'"/>'
   1.230 -                if table_mode == 'interactive':
   1.231 -                    data_results = html_functions.escape_list(row_list)
   1.232 -                    json_list = html_functions.escape(json.dumps(data_results))
   1.233 -                    content += '<input type="hidden" id="hidden_json_results" name="results" value="' + json_list  + '" />'
   1.234 +                #if table_mode == 'interactive':
   1.235 +                #    data_results = html_functions.escape_list(row_list)
   1.236 +                #    json_list = html_functions.escape(json.dumps(data_results))
   1.237 +                #    content += '<input type="hidden" id="hidden_json_results" name="results" value="' + json_list  + '" />'
   1.238                  content += '<div style="margin-bottom:5px;text-align:left;margin-left:20px;font-size:13px;color:#C2BEAD">Launch in Plotter<input type="submit" value="" /></form></div><br /><br />'
   1.239                  content += '<div style="height:33px;clear:both;text-align:left;margin-left:20px;font-size:13px;color:#C2BEAD">Query ID <a id="toggle_query_info">[+]</a><a id="toggle_query_info_help">[?]</a><div id="toggle_query_info_div" style="display:none">'+ query_results_id.split("/")[-1]  + '</div></div><br>'
   1.240  
   1.241 @@ -724,29 +748,31 @@
   1.242              content += '<div style="text-align:left;margin-left:15px;float:left;z-index:100;position: relative;clear:both"><img id ="samp_connection" src="' + survey_prefix + '/static/images/red-button.png" height="17" width="17" /><button id="register">Connect to SAMP</button><button id="unregister" style="display: none;">Disconnect from SAMP</button><button id="loadvotable" style="display: none;">Broadcast results table</button></div>'
   1.243  
   1.244              data_results = [cols]
   1.245 -            data_results.append([len(row_list)])
   1.246 +            data_results.append([row_length])
   1.247              data_results.append([content])
   1.248 +            logging.exception("Completed generate_JSON_from_query :::" +  strftime("%Y-%m-%d %H:%M:%S", gmtime()))
   1.249 +
   1.250              return json.dumps(data_results)
   1.251          else:
   1.252              return json.dumps([])
   1.253      else:
   1.254  
   1.255 -           
   1.256 -	data_results = ["ERROR"]
   1.257 -	if votable == "MAX_ERROR":
   1.258 -	    data_results.append("Query exceeded maximum return size limit")
   1.259 -	elif votable == "ERROR":
   1.260 -  	    data_results.append("There was an error processing your request")
   1.261 -	else:	
   1.262 -	    data_results.append(votable)    
   1.263 -
   1.264 -	if query_results_id!="" and query_results_id!=None:
   1.265 -	    query_id_content = '<div style="height:33px;text-align:left;margin-left:20px;font-size:13px;color:#C2BEAD">Query ID <a id="toggle_query_info">[+]</a><a id="toggle_query_info_help">[?]</a><div id="toggle_query_info_div" style="display:none"> '+ query_results_id.split("/")[-1]  + '</div></div><br>' 
   1.266 -	else :
   1.267 -	    query_id_content = ""
   1.268 -
   1.269 -	data_results.append(query_id_content)     
   1.270 -	return data_results
   1.271 +                 
   1.272 +        data_results = ["ERROR"]
   1.273 +        if votable == "MAX_ERROR":
   1.274 +            data_results.append("Query exceeded maximum return size limit")
   1.275 +        elif votable == "ERROR":
   1.276 +            data_results.append("There was an error processing your request")
   1.277 +        else:	
   1.278 +            data_results.append(votable)    
   1.279 +        
   1.280 +        if query_results_id!="" and query_results_id!=None:
   1.281 +            query_id_content = '<div style="height:33px;text-align:left;margin-left:20px;font-size:13px;color:#C2BEAD">Query ID <a id="toggle_query_info">[+]</a><a id="toggle_query_info_help">[?]</a><div id="toggle_query_info_div" style="display:none"> '+ query_results_id.split("/")[-1]  + '</div></div><br>' 
   1.282 +        else :
   1.283 +            query_id_content = ""
   1.284 +        
   1.285 +        data_results.append(query_id_content)     
   1.286 +        return data_results
   1.287  
   1.288  
   1.289  
     2.1 Binary file src/freeform_sql/misc.pyc has changed
     3.1 --- a/src/static/js/properties.js	Tue Jan 21 17:52:13 2014 +0000
     3.2 +++ b/src/static/js/properties.js	Thu Jan 30 20:43:55 2014 +0000
     3.3 @@ -1,6 +1,6 @@
     3.4  /* Global Properties */
     3.5  var sub_app_prefix='osa';
     3.6 -var base_host = 'shepseskaf.roe.ac.uk';
     3.7 +var base_host = 'localhost';
     3.8  var port = "";
     3.9  var survey_folder = 'osa';
    3.10  
     4.1 --- a/src/static/static_crossID_form/js/crossID.js	Tue Jan 21 17:52:13 2014 +0000
     4.2 +++ b/src/static/static_crossID_form/js/crossID.js	Thu Jan 30 20:43:55 2014 +0000
     4.3 @@ -17,6 +17,7 @@
     4.4  var content_div = '#content_' + this_div_name;
     4.5  var error = "There was an error processing your request";
     4.6  var gaiSelected =  [];
     4.7 +var validated_size = true;
     4.8  
     4.9  //SAMP Init Variables
    4.10  var cc;
    4.11 @@ -272,12 +273,15 @@
    4.12  	    var file = this.files[0];
    4.13  	    name = file.name;
    4.14  	    size = file.size;
    4.15 -		if (size>(1024*5)){
    4.16 +		if (size>(5242880)){
    4.17           	jQuery('#content_dbaccess_crossID_form #crossID_error').html('Upload file-size exceeded..Please try again');	
    4.18    			jQuery('#content_dbaccess_crossID_form #crossID_error').fadeIn('fast');
    4.19           	jQuery('#content_dbaccess_crossID_form #error_tooltip_wrapper').fadeIn('slow');
    4.20      		jQuery('#content_dbaccess_crossID_form #crossID_error').scrollMinimal('content_dbaccess_crossID_form');
    4.21 +    		 validated_size = false;
    4.22  
    4.23 +		} else {
    4.24 +		    validated_size = true;
    4.25  		}
    4.26  	   
    4.27  	});
    4.28 @@ -297,7 +301,13 @@
    4.29  			var validated = validate_crossID(params_for_validation);
    4.30  			jQuery('#content_dbaccess_crossID_form #error_tooltip_wrapper').hide();
    4.31  			jQuery('#content_dbaccess_crossID_form #crossID_error').hide()
    4.32 -			if (validated!='true' ){
    4.33 +			if (validated_size!=true) {
    4.34 +				jQuery('#content_dbaccess_crossID_form #crossID_error').html('Upload file-size exceeded..Please try again');	
    4.35 +      			jQuery('#content_dbaccess_crossID_form #crossID_error').fadeIn('fast');
    4.36 +             	jQuery('#content_dbaccess_crossID_form #error_tooltip_wrapper').fadeIn('slow');
    4.37 +        		jQuery('#content_dbaccess_crossID_form #crossID_error').scrollMinimal('content_dbaccess_crossID_form');
    4.38 +
    4.39 +	    	} else if (validated!='true' ){
    4.40  				
    4.41  				jQuery('#content_dbaccess_crossID_form #crossID_load').hide();
    4.42               	jQuery('#content_dbaccess_crossID_form #crossID_error').html('There was an error processing your request: ' + validated);	
     5.1 --- a/src/templates/dbaccess_login.html	Tue Jan 21 17:52:13 2014 +0000
     5.2 +++ b/src/templates/dbaccess_login.html	Thu Jan 30 20:43:55 2014 +0000
     5.3 @@ -35,6 +35,24 @@
     5.4  							
     5.5          						jQuery('#logged_in_notification').html("<br />Logged in as: <a>" + username + "</a><br />  Community: <a>" + community_input + "</a><br /><a href='" + path + "logout'>Logout</a><br /><br />");	
     5.6          						jQuery.facebox.close(); 
     5.7 +        						
     5.8 +        						jQuery.ajax({
     5.9 +                                    type: "POST",
    5.10 +                                    url: path + "getavailabledbs",
    5.11 +                                    data: {loggedin : 'true'},
    5.12 +                                    timeout: 1000000,
    5.13 +                                    error: function() {
    5.14 +                                             
    5.15 +                                    
    5.16 +                    				},
    5.17 +                    				success: function(data) {
    5.18 +                    					jQuery("select[name=database]").html(data);
    5.19 +                    					
    5.20 +                    				}
    5.21 +                    			});
    5.22 +        						
    5.23 +        						
    5.24 +        						
    5.25          					}
    5.26          				}
    5.27          			});
     6.1 --- a/src/templates/viewer.html	Tue Jan 21 17:52:13 2014 +0000
     6.2 +++ b/src/templates/viewer.html	Thu Jan 30 20:43:55 2014 +0000
     6.3 @@ -1,4 +1,4 @@
     6.4 -$def with (text, query, tap_endpoint, filepath, results, survey_prefix)
     6.5 +$def with (text, query, tap_endpoint, filepath,survey_prefix, rand)
     6.6  
     6.7  <!DOCTYPE html PUBLIC"-//W3C//DTD XHTML 1.0 Strict//EN"
     6.8  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
     6.9 @@ -85,6 +85,7 @@
    6.10  	type="text/javascript" charset="utf-8"></script>
    6.11  <script type="text/javascript">			
    6.12          
    6.13 +
    6.14          /*
    6.15           * WFAU Web2.0 Project
    6.16           * Author: Stelios Voutsinas
    6.17 @@ -297,7 +298,7 @@
    6.18      	   
    6.19            jQuery(document).ready(function(){
    6.20          	
    6.21 -        	  
    6.22 +        	  jQuery('a[rel*=facebox]').facebox();  // Initialize facebox links 
    6.23          	   /* Samp */
    6.24         
    6.25          	    cc = new samp.ClientTracker();
    6.26 @@ -584,7 +585,7 @@
    6.27  			 
    6.28          	 /* end samp */
    6.29          	  
    6.30 -         	jQuery('a[rel*=facebox]').facebox();  // Initialize facebox links 
    6.31 +         	
    6.32  
    6.33   
    6.34          	// Variables used for scatter plot 
    6.35 @@ -956,10 +957,10 @@
    6.36  
    6.37                	// Draw the individual points in areas with low density
    6.38                	for (var i = 0; i < sparsePts.length; i++) {
    6.39 -              		
    6.40 +              	
    6.41                		var xPos = chart.xAxis[0].translate(sparsePts[i][0])+chart.plotLeft;
    6.42                		var yPos = chart.plotHeight-chart.yAxis[0].translate(sparsePts[i][1])+chart.plotTop;
    6.43 -              	    primitiveObjects.push(chart.renderer.circle(xPos,yPos,3).attr({dataX : sparsePts[i][0], dataY : sparsePts[i][1], selected : 'False', fill: 'rgba(119, 152, 191, .7)',zIndex: 100}).on('click', function(){
    6.44 +              			 primitiveObjects.push(chart.renderer.circle(xPos,yPos,3).attr({dataX : sparsePts[i][0], dataY : sparsePts[i][1], selected : 'False', fill: 'rgba(119, 152, 191, .7)',zIndex: 100}).on('click', function(){
    6.45        					 	
    6.46  		              		var selected = jQuery(this)[0].attributes.selected.value;
    6.47        					 	if (selected=='True'){
    6.48 @@ -967,7 +968,7 @@
    6.49    	     				    		 fnResetAllFilters(oTable);
    6.50    	     				    		 jQuery(this)[0].attributes.selected.value = 'False';
    6.51    	     				    		 jQuery(this)[0].attributes['fill-opacity'].value = '0.6';
    6.52 -							jQuery(this)[0].attributes['fill'].value ='rgb(119,152,191)';
    6.53 +									jQuery(this)[0].attributes['fill'].value ='rgb(119,152,191)';
    6.54        					 	 } else {
    6.55        					 		 resetPointSelections();
    6.56  				        	     var x = jQuery(this)[0].attributes.dataX.value;
    6.57 @@ -1259,19 +1260,21 @@
    6.58          		fnResetAllFilters(oTable);
    6.59          		resetPointSelections();
    6.60      	    });  
    6.61 -       	  
    6.62 +        
    6.63 +       	    
    6.64              /** 
    6.65               * On plot expression link click, toggle the post-it example div 
    6.66      		 *
    6.67      		 */
    6.68 -            jQuery('#plot_expression_help_link').live('click',function(){ 
    6.69 +            jQuery('#plot_expression_help_link').live('click',function(e){ 
    6.70 +            	e.preventDefault();
    6.71      		  	if (jQuery('#plot_expression_help_link').html() == '[+]'){ 
    6.72  			  		jQuery('#plot_expression_help_link').html('[-]');
    6.73  			  	} else {
    6.74  			  		jQuery('#plot_expression_help_link').html('[+]');
    6.75  			  	}
    6.76      			jQuery('#post-it-example').slideToggle('slow');
    6.77 -			  
    6.78 +			  	return false;
    6.79        	    });
    6.80         	    
    6.81              /**
    6.82 @@ -1301,6 +1304,18 @@
    6.83      			}
    6.84         	    }); 
    6.85         	    
    6.86 +       	    jQuery('#x_axis').live('change',function(){ 
    6.87 +				
    6.88 +				if (jQuery("#x_alias").val()=="" || jQuery("#x_alias").val()==null )
    6.89 +					jQuery("#x_alias").val(jQuery("#x_axis").val());
    6.90 +       	    });
    6.91 +	       	
    6.92 +       	    jQuery('#y_axis').live('change',function(){ 
    6.93 +				
    6.94 +				if (jQuery("#y_alias").val()=="" || jQuery("#y_alias").val()==null )
    6.95 +					jQuery("#y_alias").val(jQuery("#y_axis").val());
    6.96 +    	    });
    6.97 +            
    6.98         	    /** 
    6.99         	     * On Plot Type change, adjust inputs displayed 
   6.100         	     *
   6.101 @@ -1400,7 +1415,7 @@
   6.102         					if (tables.length>0) {
   6.103  
   6.104         						var filteredArray = jQuery.map(tags, function(item) {
   6.105 -	       						if( item.startsWith( extractLast(request.term))){
   6.106 +	       						if( item.indexOf( extractLast(request.term))=== 0 ){
   6.107  						        	return item;
   6.108  						        } else {
   6.109  						            return null;
   6.110 @@ -1410,7 +1425,8 @@
   6.111         					} else {
   6.112  
   6.113  	       					var filteredArray = jQuery.map(availableTags, function(item) {
   6.114 -								if( item.startsWith( extractLast(request.term))){
   6.115 +
   6.116 +								if(  item.indexOf( extractLast(request.term))=== 0){
   6.117  						        	return item;
   6.118  						        } else {
   6.119  						            return null;
   6.120 @@ -1419,13 +1435,14 @@
   6.121         					}
   6.122         					response( jQuery.ui.autocomplete.filter(
   6.123         							filteredArray, extractLast( request.term ) ) );
   6.124 +					
   6.125         				},
   6.126         				focus: function() {
   6.127         					// prevent value inserted on focus
   6.128         					return false;
   6.129         				},
   6.130         				select: function( event, ui ) {
   6.131 -						var terms = split( this.value);
   6.132 +					var terms = split( this.value);
   6.133  
   6.134         					//var terms = this.value.split(/[\s,]+/);
   6.135         					var new_array = [];
   6.136 @@ -1450,6 +1467,16 @@
   6.137         					// add placeholder to get the comma-and-space at the end
   6.138         					terms.push( "" );
   6.139         					this.value = terms.join( " " );
   6.140 +					
   6.141 +					if (jQuery(this).attr("id")=="x_axis"){
   6.142 +						jQuery("#x_alias").val(this.value);
   6.143 +       	    				} else if (jQuery(this).attr("id")=="y_axis"){
   6.144 +						jQuery("#y_alias").val(this.value);
   6.145 +       	    				
   6.146 +					}
   6.147 +						
   6.148 +
   6.149 +
   6.150         					return false;
   6.151         				}
   6.152         			
   6.153 @@ -1559,9 +1586,12 @@
   6.154  												            jQuery('#results').fadeIn('slow');
   6.155  	  								             		    jQuery("#results .scroll-wrapper").remove();
   6.156  										                    jQuery('#results .dataTables_scrollBody').doubleScroll();
   6.157 + var row_length = oTable.fnSettings().fnRecordsTotal();
   6.158 + if (row_length>25000){  
   6.159 + 	jQuery("#length_notify").html("Plotting " + row_length + " rows with interactive plotting may be slow. We suggest you try static plotting instead" ).fadeIn();
   6.160 +}						               	}
   6.161 +								                 }
   6.162  
   6.163 -									               	}
   6.164 -								                 }
   6.165  								              }); //end ajax call
   6.166  											 
   6.167  							 } //end fnServerData function	            
   6.168 @@ -1569,12 +1599,13 @@
   6.169  			            } ); //end oTable
   6.170  
   6.171  		            }
   6.172 -			        oTable.fnAdjustColumnSizing(true);                       
   6.173 +			        oTable.fnAdjustColumnSizing(true);     
   6.174 +			       		               
   6.175  				}
   6.176         		} catch (ex){
   6.177         			alert ('There was an error processing you request');
   6.178         		}
   6.179 -       		
   6.180 +
   6.181         		
   6.182         		
   6.183              /** 
   6.184 @@ -1683,7 +1714,12 @@
   6.185                          },
   6.186                          success: function(data) {
   6.187                  			jQuery('#load').hide();  
   6.188 -                			if (data!="<div style=\"color:red;margin-left:50px;font-size:12px\">There was an error processing your request</div>"){	                   
   6.189 +                			if (data=="ERROR"){
   6.190 +                				jQuery('#error').html("There was a server error while processing your request");
   6.191 +	        					jQuery('#error').fadeIn('normal');
   6.192 +	        					jQuery('#error').delay(3800).fadeOut('slow');        
   6.193 +	        					result_array = "error";
   6.194 +                			} else if (data!="ERROR"){	                   
   6.195                  	     		jQuery('#viewer_table_div').hide();	
   6.196                  	     		jQuery('#user_table_toggle').html('[+]');
   6.197                  				jQuery('#viewer_table_div .generated_table_content').html('');
   6.198 @@ -2345,7 +2381,7 @@
   6.199  	        		}
   6.200  	        		
   6.201  	        	 	params.push({'name' : '_broadcast_url', 'value' : _broadcast_url});
   6.202 -					params.push({'name' : 'getRowIndexes', 'value' : 'true'}); 
   6.203 +				params.push({'name' : 'getRowIndexes', 'value' : 'true'}); 
   6.204  	        	 	params.push({'name' : 'pathname', 'value' : pathname});
   6.205  	    			
   6.206  	        	 	var counter = 0;
   6.207 @@ -2390,10 +2426,10 @@
   6.208  						var axis_names = jQuery("#container div.highcharts-container svg text.highcharts-title tspan").text();
   6.209  						if (axis_names.trim()!="Table-Plotter Interactive Histogram Plot" && jQuery("#mode_notify").html()=="interactive"){
   6.210  			        		axis_names = axis_names.split('  ');
   6.211 -							var x_axis = axis_names[0];
   6.212 -							var y_axis = axis_names[2];
   6.213 -							var x_axis_index = -1;
   6.214 -							var y_axis_index = -1;
   6.215 +						var x_axis = axis_names[0];
   6.216 +						var y_axis = axis_names[2];
   6.217 +						var x_axis_index = -1;
   6.218 +						var y_axis_index = -1;
   6.219  			        		var oSettings = oTable.fnSettings();  
   6.220  			        		var is_math_expr = jQuery('#is_math_expr').val();
   6.221  			        		var circle = [];
   6.222 @@ -2436,7 +2472,7 @@
   6.223  			     
   6.224  			        	 	jQuery.each(jQuery('circle'), function(i, val) {
   6.225  			        	    	 if (jQuery(this)[0].attributes.selected.value == 'True'){
   6.226 -									 jQuery('#point_notification').slideUp('slow');	 
   6.227 +								 jQuery('#point_notification').slideUp('slow');	 
   6.228  						    		 jQuery(this)[0].attributes.selected.value = 'False';
   6.229  						    		 jQuery(this)[0].attributes['fill-opacity'].value = '0.6';
   6.230  								 jQuery(this)[0].attributes['fill'].value ='rgb(119,152,191)';	
   6.231 @@ -2601,12 +2637,12 @@
   6.232  					id="filepath" $:filepath />
   6.233  				<input type="hidden" style="display: none" name="tap_endpoint"
   6.234  					id="tap_endpoint" $:tap_endpoint />
   6.235 -				<input type="hidden" style="display: none" name="results" $:results />
   6.236  				<input type="hidden" style="display: none" name="is_math_expr"
   6.237  					id="is_math_expr" value="False" />
   6.238  
   6.239  				<div style="width: 570px; margin: 10px; height: 30px;">
   6.240 -					<label class="short_label" for="plot_type">Plot type:</label> <select
   6.241 +					<label class="short_label" for="plot_type">Plot type:</label><a href="#help_plot_type" rel="facebox"
   6.242 +				style="color: gray; text-decoration: none;">[?]</a> <select
   6.243  						name="plot_type" id="plot_type"
   6.244  						style="margin-top: 5px; margin-left: 10px;">
   6.245  						<option value="Mixed">Mixed Scatter/Density</option>
   6.246 @@ -2672,7 +2708,7 @@
   6.247  		style="width: 900px; min-height: 620px; margin: 0 auto; display: none; margin-bottom: 5px"></div>
   6.248  
   6.249  	<div id="load"></div>
   6.250 -
   6.251 +	<div id="length_notify" style="display:none;color:green;margin:0 auto;text-align:center;font-style:italic;margin-top:5px;"></div>
   6.252  	<div id="density_notification"
   6.253  		style="display: none; font-size: 11px; color: green; margin: 0 auto; text-align: center; font-weight: bold"></div>
   6.254  
   6.255 @@ -2727,7 +2763,7 @@
   6.256  		</div>
   6.257  
   6.258  		<div
   6.259 -			style="margin-left: 80px; margin-top: 18px; float: left; z-index: 100; position: relative;">
   6.260 +			style="margin-left: 80px; margin-top: 18px; float: left; z-index: 45; position: relative;">
   6.261  			<img id="samp_connection"
   6.262  				src="$survey_prefix/static/images/red-button.png" height="17"
   6.263  				width="17" />
   6.264 @@ -2740,9 +2776,30 @@
   6.265  
   6.266  		<div id="results"></div> <br>
   6.267  
   6.268 +			<div id="help_plot_type" style="display: none; font-size: 13px;">
   6.269 +				<h2>Help:</h2>
   6.270 +				<br><br>
   6.271 +						<p style="font-size: 12px;">
   6.272 +							<h4>Interactive mode</h4><hr>
   6.273 +							<b>Mixed Scatter/Density plot:</b> A scatter plot where dense areas will be grouped into bins
   6.274 +							, the color of which depends on the density (number of points in that area)<br />
   6.275 +							<b>Scatter plot:</b> A simple scatter plot without any binning. (Preferable for plotting a small number of points)<br />
   6.276 +							<b>Histogram:</b> A Histogram plot with the x axis being ranges for the given column value or expression, and the y-axis being the according count<br />
   6.277 +						</p>
   6.278 +						<br><br>	
   6.279 +						<p style="font-size: 12px;">
   6.280 +							<h4>Static mode</h4><hr>
   6.281 +							<b>Density plot:</b> A denisty plot created as a static image with a color bar displaying the density colorization<br />
   6.282 +							<b>Scatter plot:</b> A simple scatter plot created as a static image<br />
   6.283 +							<b>Histogram:</b> A Histogram plot created as a static image<br />
   6.284 +						</p>
   6.285 +						
   6.286 +			</div>
   6.287 +
   6.288 +
   6.289  			<div id="help" style="display: none; font-size: 13px;">
   6.290 -				<h5>Help:</h5>
   6.291 -				<br><br>
   6.292 +				<h2>Help:</h2>
   6.293 +				<br/><br/>
   6.294  						<p style="font-size: 12px;">
   6.295  							<b>Interactive mode:</b> An interactive plot will be generated,
   6.296  							where you can zoom in and out, in a dynamic way.<br> Not
   6.297 @@ -2753,7 +2810,7 @@
   6.298  										to handle the amount of data you require to have plotted. <br><br><br><br>
   6.299  						</p>
   6.300  			</div>
   6.301 -
   6.302 +			
   6.303  			<div style="visibility: hidden">
   6.304  				<br /> <a href="http://apycom.com/">link</a><br />
   6.305  			</div>
     7.1 Binary file src/url_classes/data_tables_processing.pyc has changed
     8.1 --- a/src/url_classes/dbaccess_SQL_form.py	Tue Jan 21 17:52:13 2014 +0000
     8.2 +++ b/src/url_classes/dbaccess_SQL_form.py	Thu Jan 30 20:43:55 2014 +0000
     8.3 @@ -14,6 +14,8 @@
     8.4  import json
     8.5  import freeform_sql.firethorn_config as firethorn_config
     8.6  import urllib2
     8.7 +import time
     8.8 +from time import gmtime, strftime
     8.9  
    8.10  freeform_form = web.form.Form(web.form.Textarea('textfield',web.form.notnull,row=200,cols=30, class_='textfield', id='textfield',description=''),web.form.Hidden(name='tap_endpoint',id='tap_endpoint',value=""))
    8.11  atlasworkspace = ""
    8.12 @@ -198,8 +200,11 @@
    8.13                      for i in taps_using_binary:
    8.14                          if tap_endpoint.startswith(i):
    8.15                              _format = 'votable/td'
    8.16 -                    
    8.17 +                    logging.exception("Starting SQL query job :::" + strftime("%Y-%m-%d %H:%M:%S", gmtime()))
    8.18 +
    8.19                      results_html_val = freeform_sql.generate_JSON_from_query(freeform_sql.execute_async_query(tap_endpoint,mode_global,query,_format),query,tap_endpoint,data.table_mode)            
    8.20 +                    logging.exception("Completed SQL query job :::" + strftime("%Y-%m-%d %H:%M:%S", gmtime()))
    8.21 +
    8.22                  except Exception as e:
    8.23                      logging.exception('Exception caught:')      
    8.24                  
     9.1 Binary file src/url_classes/dbaccess_SQL_form.pyc has changed
    10.1 --- a/src/url_classes/helper_urls.py	Tue Jan 21 17:52:13 2014 +0000
    10.2 +++ b/src/url_classes/helper_urls.py	Thu Jan 30 20:43:55 2014 +0000
    10.3 @@ -7,6 +7,7 @@
    10.4  from config import mode_global, from_email, getImageURL, multiGetImageURL, \
    10.5      crossIDURL, radialURL, survey_prefix,survey_short, no_users, publicly_visible_temp_dir
    10.6  import StringIO
    10.7 +import survey_globals
    10.8  import freeform_sql
    10.9  import json
   10.10  import traceback
   10.11 @@ -17,6 +18,22 @@
   10.12  from url_classes.data_tables_processing import data_tables_processing
   10.13  
   10.14  
   10.15 +
   10.16 +class getavailabledbs:
   10.17 +
   10.18 +    def POST(self):
   10.19 +        """
   10.20 +        POST function
   10.21 +        
   10.22 +        """
   10.23 +        log_info = session.get('logged_in')
   10.24 +        if log_info == "True":  
   10.25 +            available_dbs = survey_globals.private_db_options
   10.26 +        else:
   10.27 +            available_dbs = survey_globals.db_options
   10.28 +        return available_dbs 
   10.29 +        
   10.30 +        
   10.31  class send_email:
   10.32      """ 
   10.33      Class is used as an additional URL, which is called to send an email with the results of an ADQL/TAP query
   10.34 @@ -142,7 +159,7 @@
   10.35                      form = MultipartPostHandler.MultiPartForm()
   10.36                      for i in data:
   10.37                          if i!='href' and i!='params' and i!='action' and i!='file1' and  i!='uploadFile':
   10.38 -			    logging.exception(i, data[i])
   10.39 +
   10.40                              if i=='bandarray':
   10.41  				
   10.42  				for x in data[i].split(','):
    11.1 Binary file src/url_classes/helper_urls.pyc has changed
    12.1 --- a/src/url_classes/viewer.py	Tue Jan 21 17:52:13 2014 +0000
    12.2 +++ b/src/url_classes/viewer.py	Thu Jan 30 20:43:55 2014 +0000
    12.3 @@ -54,7 +54,7 @@
    12.4          adql_table = string_functions.decode(data.adql_table)
    12.5          paren_open = ""
    12.6          paren_close = ""
    12.7 -	
    12.8 +        
    12.9          if adql_table!="" and adql_table!=None:
   12.10              query = adql_table
   12.11  
   12.12 @@ -76,11 +76,19 @@
   12.13                      _format = 'votable/td'
   12.14  
   12.15              votable,jobid, plot_file,adql_table, query_results_id = freeform_sql.execute_async_query(data.tap_endpoint,mode_global,new_query,_format) #@UnusedVariable
   12.16 -       
   12.17 +            json_data = json.loads(votable)
   12.18 +            if len(json_data)<2:
   12.19 +                votable = ""
   12.20 +            else :
   12.21 +                cols = json_data[0]
   12.22 +                row_length = len(json_data[1])
   12.23 +                rows = []
   12.24 +                for x in json_data[1]:
   12.25 +                    rows.append(x.values())
   12.26              if votable!='':            
   12.27 -                if len(votable)>0:
   12.28 +                if row_length>0:
   12.29                      try:
   12.30 -                        data_to_list = votable.data.tolist()     
   12.31 +                        data_to_list = rows     
   12.32                          if (data.plot_type=="Scatter" or data.plot_type == "Density" or data.plot_type == "Mixed"):
   12.33                              if (type (data_to_list[0][0]) is str) or (type (data_to_list[0][1]) is str):
   12.34                                  data_to_list = [map (self.tryeval,x) for x in data_to_list]
   12.35 @@ -92,7 +100,7 @@
   12.36                                  if (type (data_to_list[0][0]) is str):
   12.37                                      raise Exception
   12.38                                  
   12.39 -                        data_results = [list(votable.columns)]
   12.40 +                        data_results = [cols]
   12.41                          data_results.append(data_to_list)
   12.42                          
   12.43                          if data.mode=="static":    
   12.44 @@ -109,17 +117,16 @@
   12.45                          
   12.46                      except Exception as e:
   12.47                          logging.exception('Exception caught:')
   12.48 -                        return_value = "<div style=\"color:red;margin-left:50px;font-size:12px\">There was an error processing your request</div>"   
   12.49 +                        return_value = "ERROR"   
   12.50              else : 
   12.51 -                return_value = "<div style=\"color:red;margin-left:50px;font-size:12px\">There was an error processing your request</div>"    
   12.52 +                return_value = "ERROR"    
   12.53          table_cols = data.cols
   12.54          
   12.55          if table_cols == None or table_cols == "":
   12.56              table_cols = 0
   12.57 -        
   12.58 +       
   12.59          if return_value=='':
   12.60 -       
   12.61 -            return render.viewer(table_cols,'value="'+ query + '"','value="'+ data.tap_endpoint + '"','value="'+ data.filepath + '"', 'value="' + freeform_sql.html_functions.escape(data.results) +'"', survey_prefix, rand)   
   12.62 +            return render.viewer(table_cols,'value="'+ query + '"','value="'+ data.tap_endpoint + '"','value="'+ data.filepath + '"', survey_prefix, rand)   
   12.63          else :
   12.64              return return_value
   12.65  
    13.1 Binary file src/url_classes/viewer.pyc has changed