From a43e121eb1357588734db23948ad3e97d229171d Mon Sep 17 00:00:00 2001
From: Marcus O'Flaherty <marcus.o-flaherty@warwick.ac.uk>
Date: Fri, 14 Mar 2025 11:38:34 +0000
Subject: [PATCH 1/7] add removal of stale endpoints from connections map add
 loop timers and checking for slow loops in Execute run all read queries then
 all write queries, only one Multicast query per loop. Poor man's message
 prioritising as read/writes have a timeout and some multicast (monitoring)
 queries were taking very long (several seconds) to execute. change some
 verbositiies.

---
 MMUtilities.cpp  |  64 ++++++++++++++++++++--------
 Postgres.cpp     |   1 +
 ReceiveSQL.cpp   | 107 ++++++++++++++++++++++++++++++++++++++++-------
 ReceiveSQL.h     |   5 +++
 ReceiveSQLConfig |   4 +-
 show_logs.sh     |   7 ++++
 6 files changed, 153 insertions(+), 35 deletions(-)
 create mode 100755 show_logs.sh

diff --git a/MMUtilities.cpp b/MMUtilities.cpp
index 19d2b95..d4c757d 100644
--- a/MMUtilities.cpp
+++ b/MMUtilities.cpp
@@ -14,9 +14,6 @@ int MMUtilities::ConnectToEndpoints(zmq::socket_t* readrep_sock, std::map<std::s
     // for middlemen, we likewise find their Control service and assume they have a inter-middlemen comms point
     // since we connect to hidden endpoints, we need to already know their port numbers
     
-    boost::uuids::uuid m_UUID=boost::uuids::random_generator()();
-    long msg_id=0;
-
     zmq::socket_t Ireceive (*m_context, ZMQ_DEALER);
     Ireceive.connect("inproc://ServiceDiscovery");
 
@@ -39,6 +36,8 @@ int MMUtilities::ConnectToEndpoints(zmq::socket_t* readrep_sock, std::map<std::s
     iss>>size;
     
     int num_new_connections=0;
+    
+    std::set<std::string> active_endpoints;
 
     for(int i=0;i<size;i++){
       
@@ -61,6 +60,8 @@ int MMUtilities::ConnectToEndpoints(zmq::socket_t* readrep_sock, std::map<std::s
       std::string tmp;
       bool registered=false;
       
+      active_endpoints.emplace(ip);
+      
       if(type.substr(0,9)!="middleman"){
         // if this isn't a middleman, assume it's a service with a PGClient
         // try to connect to the standard PGClient ports
@@ -83,14 +84,13 @@ int MMUtilities::ConnectToEndpoints(zmq::socket_t* readrep_sock, std::map<std::s
           // at the least we should check the currently set value represents one that
           // will be set by zmq::socket_t::connect in the event of an error
           errno=0;
-          readrep_sock->connect(tmp.c_str());
-          if(errno!=0 && connect_errs.count(errno)){
-            std::cerr<<"MMUtilities::ConnectToEndpoints error connecting to read socket "
-                     <<tmp<<": "<<zmq_strerror(errno)<<std::endl;
-          } else {
+          try {
+            readrep_sock->connect(tmp.c_str());
             std::cout<<"MMUtilities::ConnectToEndpoints new connection to read socket "<<tmp<<std::endl;
+            ++num_new_connections;
+          } catch(zmq::error_t& err){
+            std::cerr<<"MMUtilities::ConnectToEndpoints error connecting to read socket "<<tmp<<": "<<err.what()<<std::endl;
           }
-          ++num_new_connections;
           
           // write socket is only connected to by the master middleman
           if(write_sock){
@@ -103,14 +103,13 @@ int MMUtilities::ConnectToEndpoints(zmq::socket_t* readrep_sock, std::map<std::s
             tmp=ip + ":" + store_port;
             tmp="tcp://"+ tmp;
             errno=0;
-            write_sock->connect(tmp.c_str());
-            if(errno!=0 && connect_errs.count(errno)){
-              std::cerr<<"MMUtilities::ConnectToEndpoints error connecting to write socket "
-                       <<tmp<<": "<<zmq_strerror(errno)<<std::endl;
-            } else {
+            try {
+              write_sock->connect(tmp.c_str()); // no return value but will throw instead!
               std::cout<<"MMUtilities::ConnectToEndpoints new connection to write socket "<<tmp<<std::endl;
+              ++num_new_connections;
+            } catch(zmq::error_t& err){
+              std::cerr<<"MMUtilities::ConnectToEndpoints error connecting to write socket "<<tmp<<": "<<err.what()<<std::endl;
             }
-            ++num_new_connections;
           }
           
         } // else we're already connected to this service
@@ -127,17 +126,48 @@ int MMUtilities::ConnectToEndpoints(zmq::socket_t* readrep_sock, std::map<std::s
           mm_conns[ip]=service;
           tmp=ip + ":" + store_port;
           tmp="tcp://"+ tmp;
-          mm_sock->connect(tmp.c_str());
-          ++num_new_connections;
+          try {
+            mm_sock->connect(tmp.c_str());
+            std::cout<<"MMUtilities::ConnectToEndpoints new connection to middleman "<<tmp<<std::endl;
+            ++num_new_connections;
+          } catch(zmq::error_t& err){
+            std::cerr<<"MMUtilities::ConnectToEndpoints error connecting to middleman "<<tmp<<": "<<err.what()<<std::endl;
+          }
         }
         
       }
       
+      // delete the Store if we're not keeping it
       if(!registered){
         delete service;
         service=0;
       }
       
+    } // end loop over services in broadcast
+    
+    // prune inactive endpoints
+    std::vector<std::map<std::string,Store*>::iterator> to_erase;
+    for(std::map<std::string,Store*>::iterator it=readrep_conns.begin(); it!=readrep_conns.end(); ++it){
+      if(active_endpoints.count(it->first)==0){
+        std::cout<<"Booting inactive endpoint "<<it->first<<std::endl;
+        /*
+        // could explicitly disconnect?
+        std::string store_port="";
+        it->second->Get("remote_port",store_port);
+        std::string tmp="tcp://"+ it->first + ":" + store_port;
+        try {
+          readrep_sock->disconnect(tmp.c_str());
+          write_sock->disconnect(tmp.c_str());
+        } catch(zmq::error_t& err){
+          std::cerr<<"MMUtilities::ConnectToEndpoints error disconnecting from stale socket "<<tmp<<": "<<err.what()<<std::endl;
+        }
+        */
+        delete it->second;
+        to_erase.push_back(it);
+      }
+    }
+    for(std::map<std::string,Store*>::iterator it : to_erase){
+      readrep_conns.erase(it);
     }
     
     return num_new_connections;
diff --git a/Postgres.cpp b/Postgres.cpp
index d2e626c..89be8cc 100755
--- a/Postgres.cpp
+++ b/Postgres.cpp
@@ -118,6 +118,7 @@ void Postgres::Init(std::string hostname_in, std::string hostip_in, int port_in,
 
 // XXX reminder that pqxx::result is a reference-counting wrapper and is not thread-safe! XXX
 bool Postgres::Query(std::string query, int nret, pqxx::result* res, pqxx::row* row, std::string* err){
+	
 	// maybe this is redundant since OpenConnection will check is_open (against recommendations)
 	for(int tries=0; tries<2; ++tries){
 		// ensure we have a connection to work with
diff --git a/ReceiveSQL.cpp b/ReceiveSQL.cpp
index f595093..2582745 100755
--- a/ReceiveSQL.cpp
+++ b/ReceiveSQL.cpp
@@ -56,11 +56,16 @@ bool ReceiveSQL::Initialise(const std::string& configfile){
 
 bool ReceiveSQL::Execute(){
 	Log("ReceiveSQL Executing...",21);
+	
 	auto loop_start = std::chrono::high_resolution_clock::now();
+	auto last = std::chrono::high_resolution_clock::now();
 	
 	// find new clients
 	Log("Finding new clients",20);
 	get_ok = FindNewClients_v2();
+	timers["Finding Clients"] =
+	               std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - last).count();
+	last = std::chrono::high_resolution_clock::now();
 	
 	// poll the input sockets for messages
 	Log("Polling input sockets",20);
@@ -74,6 +79,10 @@ bool ReceiveSQL::Execute(){
 		Log("Warning! ReceiveSQL error polling input sockets; have they closed?",0);
 	}
 	
+	timers["Poll input sockets"] =
+	               std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - last).count();
+	last = std::chrono::high_resolution_clock::now();
+	
 	// debug print
 	std::string pollsmsg;
 	for(int i=0; i<in_polls.size(); ++i){
@@ -106,26 +115,57 @@ bool ReceiveSQL::Execute(){
 		Log("Getting Client Write Queries",20);
 		get_ok = GetClientWriteQueries();
 	}
+	timers["Get Write Queries"] =
+	               std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - last).count();
+	last = std::chrono::high_resolution_clock::now();
+	
 	Log("Getting Client Read Queries",20);
 	get_ok = GetClientReadQueries();
+	timers["Get Read Queries"] =
+	               std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - last).count();
+	last = std::chrono::high_resolution_clock::now();
+	
 	if(am_master){
 		Log("Getting Client Multicast Messages",20);
 		get_ok = GetMulticastMessages();
 	}
+	timers["Get Multicasts"] =
+	               std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - last).count();
+	last = std::chrono::high_resolution_clock::now();
+	
 	Log("Getting Middleman Checkin",20);
 	get_ok = GetMiddlemanCheckin();
+	timers["Get MM check-in"] =
+	               std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - last).count();
+	last = std::chrono::high_resolution_clock::now();
+	
 	Log("Checking Master Status",20);
 	get_ok = CheckMasterStatus();
+	timers["Get MM status"] =
+	               std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - last).count();
+	last = std::chrono::high_resolution_clock::now();
+	
 	if(am_master){
 		Log("Running Next Write Query",20);
 		get_ok = RunNextWriteQuery();
 	}
+	timers["Run Write Query"] =
+	               std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - last).count();
+	last = std::chrono::high_resolution_clock::now();
+	
 	Log("Running Next Read Query",20);
 	get_ok = RunNextReadQuery();
+	timers["Run Read Query"] =
+	               std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - last).count();
+	last = std::chrono::high_resolution_clock::now();
+	
 	if(am_master){
 		Log("Running Next Fire-and-Forget Message",20);
 		get_ok = RunNextMulticastMsg();
 	}
+	timers["Run Multicast"] =
+	               std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - last).count();
+	last = std::chrono::high_resolution_clock::now();
 	
 	// poll the output sockets for listeners
 	Log("Polling output sockets",20);
@@ -138,14 +178,28 @@ bool ReceiveSQL::Execute(){
 	if(get_ok<0){
 		Log("Warning! ReceiveSQL error polling output sockets; have they closed?",0);
 	}
+	timers["Poll output sockets"] =
+	               std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - last).count();
+	last = std::chrono::high_resolution_clock::now();
 	
 	// send outputs
 	Log("Sending Next Client Response",20);
 	get_ok = SendNextReply();
+	timers["Send Reply"] =
+	               std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - last).count();
+	last = std::chrono::high_resolution_clock::now();
+	
 	Log("Sending Next Log Message",20);
 	get_ok = SendNextMulticast();
+	timers["Send Multicast"] =
+	               std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - last).count();
+	last = std::chrono::high_resolution_clock::now();
+	
 	Log("Broadcasting Presence",20);
 	get_ok = BroadcastPresence();
+	timers["SD Broadcast"] =
+	               std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - last).count();
+	last = std::chrono::high_resolution_clock::now();
 	
 	// Maintenance
 	Log("Trimming Write Queue",20);
@@ -162,14 +216,23 @@ bool ReceiveSQL::Execute(){
 	get_ok = TrimCache();
 	Log("Cleaning Up Old Cache Messages",20);
 	get_ok = CleanupCache();
+	timers["Cleanup"] =
+	               std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - last).count();
+	last = std::chrono::high_resolution_clock::now();
 	
 	// Monitoring and Logging
 	Log("Tracking Stats",20);
 	if(!stats_period.is_negative()) get_ok = TrackStats();
+	timers["Stats"] =
+	               std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - last).count();
+	last = std::chrono::high_resolution_clock::now();
 	
 	// Check for any commands from remote control port
 	Log("Checking Controls",20);
 	get_ok = UpdateControls();
+	timers["Update Controls"] =
+	               std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - last).count();
+	last = std::chrono::high_resolution_clock::now();
 	
 	Log("Loop Iteration Done",20);
 	auto loop_end = std::chrono::high_resolution_clock::now();
@@ -178,6 +241,11 @@ bool ReceiveSQL::Execute(){
 	int loop_ms = std::chrono::duration_cast<std::chrono::milliseconds>(loop_end - loop_start).count();
 	if(loop_ms>3000){
 		Log("Warning: Middleman Execute took "+std::to_string(loop_ms)+"ms!",v_warning);
+		std::string msg="Timers were: ";
+		for(auto&& atimer : timers){
+			msg+= atimer.first + ": "+std::to_string(atimer.second)+" ms\n";
+		}
+		Log(msg,v_warning);
 	}
 	if(loop_ms<min_loop_ms) min_loop_ms=loop_ms;
 	if(loop_ms>max_loop_ms) max_loop_ms=loop_ms;
@@ -822,9 +890,9 @@ bool ReceiveSQL::FindNewClients_v2(){
 		
 		std::string clientlist;
 		for(std::pair<const std::string,std::string>& aclient : clientsmap){
-			clientlist += aclient.first+": "+aclient.second+",";
+			if(!clientlist.empty()) clientlist+=", ";
+			clientlist += aclient.first+": "+aclient.second;
 		}
-		clientlist.pop_back(); // remove trailing ','
 		SC_vars["Clients"]->SetValue(clientlist);
 		
 	} else {
@@ -1219,7 +1287,7 @@ bool ReceiveSQL::WriteRootPlotToQuery(const std::string& message, BStore& plot,
 		+ draw_options  + ","
 		+ json_data       + ") returning version;";
 	
-	Log(Concat("Resulting SQL: '",sql_out,"', database: '",db_out,"'"),4);
+	Log(Concat("Resulting SQL: '",sql_out,"', database: '",db_out,"'"),50);
 	
 	return true;
 	
@@ -1267,11 +1335,11 @@ bool ReceiveSQL::WritePlotToQuery(const std::string& message, BStore& plot, std:
 		" SET (x, y, title, xlabel, ylabel, info) = row("
 		"   EXCLUDED.x, EXCLUDED.y, EXCLUDED.title, EXCLUDED.xlabel, EXCLUDED.ylabel, EXCLUDED.info"
 		" );";
-
-	Log(Concat("Resulting SQL: '", sql_out, "', database: '", db_out, "'"), 4);
-
+	
+	Log(Concat("Resulting SQL: '", sql_out, "', database: '", db_out, "'"), 40);
+	
 	return true;
-
+	
 fail_get:
 	get_ok = 0;
 	Log("WritePlotToQuery: missing fields in message '" + message + "'", v_error);
@@ -1454,7 +1522,7 @@ bool ReceiveSQL::ReadDeviceConfigToQuery(const std::string& message, BStore& req
 		+ device + " AND version="
 		+ versionstring+";";
 	
-	Log(Concat("Resulting SQL: '",sql_out,"', database: '",db_out,"'"),4);
+	Log(Concat("Resulting SQL: '",sql_out,"', database: '",db_out,"'"),12);
 	
 	return true;
 }
@@ -1515,7 +1583,7 @@ bool ReceiveSQL::ReadRunConfigToQuery(const std::string& message, BStore& reques
 		
 	}
 	
-	Log(Concat("Resulting SQL: '",sql_out,"', database: '",db_out,"'"),4);
+	Log(Concat("Resulting SQL: '",sql_out,"', database: '",db_out,"'"),12);
 	
 	return true;
 }
@@ -1555,7 +1623,7 @@ bool ReceiveSQL::ReadCalibrationToQuery(const std::string& message, BStore& requ
 		+ device + " AND version="
 		+ versionstring+";";
 	
-	Log(Concat("Resulting SQL: '",sql_out,"', database: '",db_out,"'"),4);
+	Log(Concat("Resulting SQL: '",sql_out,"', database: '",db_out,"'"),12);
 	
 	return true;
 	
@@ -1592,7 +1660,7 @@ bool ReceiveSQL::ReadRootPlotToQuery(const std::string& message, BStore& request
 		sql_out += " AND version=" + std::to_string(version);
 	}
 	
-	Log(Concat("Resulting SQL: '",sql_out,"', database: '",db_out,"'"),4);
+	Log(Concat("Resulting SQL: '",sql_out,"', database: '",db_out,"'"),40);
 	
 	return true;
 	
@@ -1621,7 +1689,7 @@ bool ReceiveSQL::ReadPlotToQuery(const std::string& message, BStore& request, st
 
 	sql_out = "SELECT x, y, title, xlabel, ylabel, info FROM plots WHERE plot = " + name + ";";
 
-	Log(Concat("Resulting SQL: '", sql_out, "', database: '", db_out, "'"), 4);
+	Log(Concat("Resulting SQL: '", sql_out, "', database: '", db_out, "'"), 40);
 
 	return true;
 }
@@ -1780,7 +1848,7 @@ bool ReceiveSQL::MulticastMessageToQuery(const std::string& message, std::string
 		        + std::to_string(severity) + ","
 		        + msg                      + ");";
 		
-		Log(Concat("Resulting SQL: '",sql_out,"', database: '",db_out,"', topic: ",topic_out),12);
+		Log(Concat("Resulting SQL: '",sql_out,"', database: '",db_out,"', topic: ",topic_out),100);
 		
 		return true;
 		
@@ -1822,7 +1890,7 @@ bool ReceiveSQL::MulticastMessageToQuery(const std::string& message, std::string
 		        + device     + ","
 		        + data       + ");";
 		
-		Log(Concat("Resulting SQL: '",sql_out,"', database: '",db_out,"', topic: ",topic_out),12);
+		Log(Concat("Resulting SQL: '",sql_out,"', database: '",db_out,"', topic: ",topic_out),50);
 		
 		return true;
 		
@@ -1870,7 +1938,7 @@ bool ReceiveSQL::MulticastMessageToQuery(const std::string& message, std::string
 		        + draw_options  + ","
 		        + data       + ");";
 		
-		Log(Concat("Resulting SQL: '",sql_out,"', database: '",db_out,"', topic: ",topic_out),12);
+		Log(Concat("Resulting SQL: '",sql_out,"', database: '",db_out,"', topic: ",topic_out),50);
 		
 		return true;
 		
@@ -2094,7 +2162,8 @@ bool ReceiveSQL::RunNextWriteQuery(){
 bool ReceiveSQL::RunNextReadQuery(){
 	
 	// run our next postgres query, if we have one
-	if(rd_txn_queue.size()){
+	//if(rd_txn_queue.size()){
+	while(rd_txn_queue.size()){
 		
 		Query& next_msg = rd_txn_queue.begin()->second;
 		std::string& db = next_msg.database;
@@ -2130,7 +2199,13 @@ bool ReceiveSQL::RunNextMulticastMsg(){
 		    +" messages to process",13);
 		
 		std::string next_msg = in_multicast_queue.front();
+		auto t_start = std::chrono::high_resolution_clock::now();
 		get_ok = m_databases.at("daq").Query(next_msg);  // FIXME hard-coded db name
+		auto t_end = std::chrono::high_resolution_clock::now();
+		if(std::chrono::duration_cast<std::chrono::milliseconds>(t_end-t_start).count()>1000){
+			Log("Slow multicast query: '"+next_msg+"' took "+
+			std::to_string(std::chrono::duration_cast<std::chrono::milliseconds>(t_end-t_start).count())+" ms",v_warning);
+		}
 		
 		if(not get_ok){
 			// something went wrong
diff --git a/ReceiveSQL.h b/ReceiveSQL.h
index b623b45..c6a62d3 100755
--- a/ReceiveSQL.h
+++ b/ReceiveSQL.h
@@ -26,6 +26,8 @@
 #include <string>
 #include <iostream>
 #include <deque>
+//#include <unordered_map>
+#include <map>
 // multicast
 #include <sys/socket.h>
 #include <sys/types.h>
@@ -285,6 +287,9 @@ class ReceiveSQL{
 	// for holding stats variables and turning them into a json
 	Store MonitoringStore;
 	
+	// tracking where we spend our time
+	std::map<std::string, int> timers; // function -> time in ms
+	
 	////////////
 	// variadic templates: our excuse to use c++11 ;)
 	
diff --git a/ReceiveSQLConfig b/ReceiveSQLConfig
index 213caab..cfbb7ca 100644
--- a/ReceiveSQLConfig
+++ b/ReceiveSQLConfig
@@ -41,13 +41,13 @@ dont_promote 0
 max_send_attempts 3
 # how many postgres queries in the queue before we warn about being full
 warn_limit 700
-# how many postgres queries before we start dropping them 
+# how many postgres queries before we start dropping them
 drop_limit 1000
 # how long to cache received messages so we don't get duplicates
 cache_period_ms 10000
 # if writes are sent to the read-only port, but we're master, should we accept them?
 handle_unexpected_writes 0
 stats_period_ms 60000
-stdio_verbosity 6
+stdio_verbosity 3
 db_verbosity 1
 service_discovery_config ServiceDiscoveryConfig
diff --git a/show_logs.sh b/show_logs.sh
new file mode 100755
index 0000000..af836ea
--- /dev/null
+++ b/show_logs.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+if [ $# -gt 0 ]; then
+	if [ "$1" == "-f" ]; then
+		journalctl -f  _SYSTEMD_INVOCATION_ID=`systemctl show --value -p InvocationID middleman`
+	fi
+fi
+journalctl  _SYSTEMD_INVOCATION_ID=`systemctl show --value -p InvocationID middleman` | less

From dfce780f6c3808fecbceb5bb69a8048fd43b2494 Mon Sep 17 00:00:00 2001
From: Marcus O'Flaherty <marcus.o-flaherty@warwick.ac.uk>
Date: Wed, 26 Mar 2025 15:41:15 +0000
Subject: [PATCH 2/7] add systemd units, sanitizer to makefile, remove comment
 about zmqcpp not returning error code as it throws exception instead

---
 .gitignore              |  1 +
 MMUtilities.cpp         |  8 ------
 Makefile                |  2 +-
 middleman.service       | 23 +++++++++++++++++
 notify-service@.service | 11 ++++++++
 slack_notify.sh         | 56 +++++++++++++++++++++++++++++++++++++++++
 6 files changed, 92 insertions(+), 9 deletions(-)
 create mode 100644 middleman.service
 create mode 100644 notify-service@.service
 create mode 100755 slack_notify.sh

diff --git a/.gitignore b/.gitignore
index 9ba131a..2719f45 100644
--- a/.gitignore
+++ b/.gitignore
@@ -19,3 +19,4 @@ middleman
 middleman_runs.log
 stop
 quit
+slack_web_hook.txt
diff --git a/MMUtilities.cpp b/MMUtilities.cpp
index d4c757d..7adfde6 100644
--- a/MMUtilities.cpp
+++ b/MMUtilities.cpp
@@ -76,14 +76,6 @@ int MMUtilities::ConnectToEndpoints(zmq::socket_t* readrep_sock, std::map<std::s
           readrep_conns[ip]=service;
           tmp=ip + ":" + store_port;
           tmp="tcp://"+ tmp;
-          // annoyingly libzmq doesn't expose the return value of zmq::socket_t::connect
-          // so we can't clearly check whether a socket connect succeeded or failed. >:(
-          // if it fails it should set errno to say why. Now it is poor practice to use
-          // errno unless you know it should be checked, because its value may be set by
-          // intervening calls that do not indicate an actual error. But its the best we have.
-          // at the least we should check the currently set value represents one that
-          // will be set by zmq::socket_t::connect in the event of an error
-          errno=0;
           try {
             readrep_sock->connect(tmp.c_str());
             std::cout<<"MMUtilities::ConnectToEndpoints new connection to read socket "<<tmp<<std::endl;
diff --git a/Makefile b/Makefile
index c387780..b92036d 100755
--- a/Makefile
+++ b/Makefile
@@ -16,7 +16,7 @@ ToolFrameworkInclude= -I $(Dependencies)/ToolFrameworkCore/include
 ToolDAQFrameworkLib=  -L $(Dependencies)/ToolDAQFramework/lib -lDAQStore -lDAQDataModelBase -lServiceDiscovery
 ToolDAQFrameworkInclude= -I $(Dependencies)/ToolDAQFramework/include
 
-CXXFLAGS= -g -fdiagnostics-color=always -Wno-attributes -O3
+CXXFLAGS= -g -fdiagnostics-color=always -Wno-attributes -O3 -fsanitize=address
 
 all: middleman
 
diff --git a/middleman.service b/middleman.service
new file mode 100644
index 0000000..ba7eaf2
--- /dev/null
+++ b/middleman.service
@@ -0,0 +1,23 @@
+[Unit]
+Description=middleman application that interfaces with the database via zmq
+After=postgresql.service
+Requires=postgresql.service
+OnFailure=notify-slack@%i.service
+
+[Service]
+Type=simple
+#Environment="PGHOST=/var/run/psql"  # some debian-based distros use this, though it can be configured in postgresql.conf
+Environment="PGHOST=/tmp"
+Environment="PGUSER=root"
+Environment="PGPORT=5432"
+Environment="PGDATABASE=daq"
+Environment="PGDATA=/var/lib/pgsql/data"
+Environment="LD_LIBRARY_PATH=/opt/zeromq-4.0.7/lib:/opt/boost_1_66_0/install/lib:/opt/libpqxx-6.4.5/install/lib:/opt/ToolFrameworkCore/lib:/opt/ToolDAQFramework/lib:$LD_LIBRARY_PATH"
+Environment="ASAN_OPTIONS=allocator_may_return_null=1"
+WorkingDirectory=/opt/middleman
+ExecStart=/opt/middleman/middleman /opt/middleman/ReceiveSQLConfig
+# if the middleman stops without the user doing so via systemctl, restart it automatically
+Restart=always
+
+[Install]
+WantedBy=default.target
diff --git a/notify-service@.service b/notify-service@.service
new file mode 100644
index 0000000..f88be2c
--- /dev/null
+++ b/notify-service@.service
@@ -0,0 +1,11 @@
+[Unit]
+Description=Send Notification of systemd unit Failure
+
+[Service]
+Type=oneshot
+#ExecStart=echo 'Notification triggered for service %i'
+ExecStart=/opt/middleman/slackNotify.sh -s %i
+
+[Install]
+WantedBy=default.target
+EOF
diff --git a/slack_notify.sh b/slack_notify.sh
new file mode 100755
index 0000000..5eb9fe7
--- /dev/null
+++ b/slack_notify.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+# Bash script to send systemd notifications to Slack
+
+function usage {
+    programName=$0
+    echo "description: use this script to post systemd service failure message to Slack channel"
+    echo "usage: $programName -s \"service name\""
+    echo "	-s    the systemd service name e.g. nginx"
+    exit 1
+}
+
+# Get service name from options
+while getopts ":s:" opt; do
+  case $opt in
+    s)
+      SERVICE_NAME=$OPTARG
+      ;;
+    \?)
+      echo "Invalid option: -$OPTARG" >&2
+      exit 1
+      ;;
+    :)
+      echo "Option -$OPTARG requires an argument." >&2
+      exit 1
+      ;;
+  esac
+done
+
+#echo "service name is '${SERVICE_NAME}'"
+
+if [[ ! "${SERVICE_NAME}" ]]; then
+    echo "Service name is required"
+    usage
+fi
+
+# Edit the following variables to match your requirements
+SLACK_HOOK_FILE=/opt/middleman/slack_web_hook.txt
+SLACK_WEBHOOK_URL=$(cat ${SLACK_HOOK_FILE})
+SLACK_CHANNEL="#general"
+SLACK_USERNAME="Notification Bot"
+SLACK_ICON=":zap:"
+SLACK_COLOR="danger"
+SLACK_TITLE="Service $SERVICE_NAME failed on $(hostname)"
+SLACK_PRETEXT="Service $SERVICE_NAME failed"
+SLACK_TEXT="$(systemctl status $SERVICE_NAME)"
+#SLACK_TEXT="testing service \"potato\""
+SLACK_FOOTER="Notification Bot at $(hostname) on $(date)"
+# End of variables
+
+SLACK_ATTACHMENT='[{"fallback": "'"$SLACK_MESSAGE"'", "color": "'"$SLACK_COLOR"'", "title": "'"$SLACK_TITLE"'", "title_link": "'"$SLACK_TITLE_LINK"'", "pretext": "'"$SLACK_PRETEXT"'", "text": "'"$SLACK_TEXT"'", "footer": "'"$SLACK_FOOTER"'", "footer_icon": "'"$SLACK_FOOTER_ICON"'"}]'
+
+# Send notification to Slack
+curl -X POST --data-urlencode 'payload={"channel": "'"$SLACK_CHANNEL"'", "username": "'"$SLACK_USERNAME"'", "icon_emoji": "'"$SLACK_ICON"'", "attachments": '"$SLACK_ATTACHMENT"'}' $SLACK_WEBHOOK_URL
+
+# Exit with success code
+exit 0

From a15ce06ea0b66316c0b9a9394623aa3cf3284e43 Mon Sep 17 00:00:00 2001
From: Marcus O'Flaherty <marcus.o-flaherty@warwick.ac.uk>
Date: Thu, 5 Jun 2025 15:35:55 +0000
Subject: [PATCH 3/7] Add multithreading to middleman. add mutex locking to zmq
 sockets in MMUtilities::ConnectToEndpoints extend Postgres class to allow
 users to pass in a pqxx::connection, while also retaining one member one.
 This is needed if using multiple threads as each needs their own connection.
 add topic to Query class, as it now gets forwarded to worker threads that do
 SQL query interpretation

---
 MMUtilities.cpp    |  43 ++-
 MMUtilities.h      |   5 +-
 Postgres.cpp       | 109 ++++---
 Postgres.h         |  39 +--
 Query.cpp          |   4 +-
 Query.h            |   3 +-
 ReceiveSQL.cpp     | 698 ++++++++++++++++++++++++++++++++-------------
 ReceiveSQL.h       |  91 +++++-
 ReceiveSQLConfig   |   5 +-
 run_maintenance.sh |  14 +
 10 files changed, 725 insertions(+), 286 deletions(-)
 create mode 100755 run_maintenance.sh

diff --git a/MMUtilities.cpp b/MMUtilities.cpp
index 7adfde6..16bc936 100644
--- a/MMUtilities.cpp
+++ b/MMUtilities.cpp
@@ -8,11 +8,15 @@ MMUtilities::MMUtilities(zmq::context_t* zmqcontext) : DAQUtilities(zmqcontext){
 
 const std::set<int> connect_errs{EINVAL,EPROTONOSUPPORT,ENOCOMPATPROTO,ETERM,ENOTSOCK,EMTHREAD};
 
-int MMUtilities::ConnectToEndpoints(zmq::socket_t* readrep_sock, std::map<std::string,Store*> &readrep_conns, int read_port_num, zmq::socket_t* write_sock, std::map<std::string,Store*> &write_conns, int write_port_num, zmq::socket_t* mm_sock, std::map<std::string, Store*> &mm_conns, int mm_port_num){
+int MMUtilities::ConnectToEndpoints(zmq::socket_t* readrep_sock, std::map<std::string,Store*> &readrep_conns, int read_port_num, std::mutex& readrep_mtx, zmq::socket_t* write_sock, std::map<std::string,Store*> &write_conns, int write_port_num, std::mutex& write_mtx, zmq::socket_t* mm_sock, std::map<std::string, Store*> &mm_conns, int mm_port_num, std::mutex& mm_mtx){
     // it's like UpdateConnections, but rather than connecting to specifically named endpoints,
     // we find all services that aren't middlemen and assume they have associated postgres client endpoints
     // for middlemen, we likewise find their Control service and assume they have a inter-middlemen comms point
     // since we connect to hidden endpoints, we need to already know their port numbers
+    if(readrep_sock==nullptr || write_sock==nullptr || mm_sock==nullptr){
+        std::cerr<<"ConnectToEndpoints with null sockets: "<<readrep_sock<<", "<<write_sock<<", "<<mm_sock<<std::endl;
+        return 0;
+    }
     
     zmq::socket_t Ireceive (*m_context, ZMQ_DEALER);
     Ireceive.connect("inproc://ServiceDiscovery");
@@ -50,11 +54,9 @@ int MMUtilities::ConnectToEndpoints(zmq::socket_t* readrep_sock, std::map<std::s
       service->JsonParser(ss.str());
       
       std::string type;
-      std::string uuid;
       std::string ip;
       std::string store_port="";
       service->Get("msg_value",type);
-      service->Get("uuid",uuid);
       service->Get("ip",ip);
       service->Get("remote_port",store_port);
       std::string tmp;
@@ -77,7 +79,9 @@ int MMUtilities::ConnectToEndpoints(zmq::socket_t* readrep_sock, std::map<std::s
           tmp=ip + ":" + store_port;
           tmp="tcp://"+ tmp;
           try {
+            readrep_mtx.lock();
             readrep_sock->connect(tmp.c_str());
+            readrep_mtx.unlock();
             std::cout<<"MMUtilities::ConnectToEndpoints new connection to read socket "<<tmp<<std::endl;
             ++num_new_connections;
           } catch(zmq::error_t& err){
@@ -96,7 +100,9 @@ int MMUtilities::ConnectToEndpoints(zmq::socket_t* readrep_sock, std::map<std::s
             tmp="tcp://"+ tmp;
             errno=0;
             try {
+              write_mtx.lock();
               write_sock->connect(tmp.c_str()); // no return value but will throw instead!
+              write_mtx.unlock();
               std::cout<<"MMUtilities::ConnectToEndpoints new connection to write socket "<<tmp<<std::endl;
               ++num_new_connections;
             } catch(zmq::error_t& err){
@@ -119,7 +125,9 @@ int MMUtilities::ConnectToEndpoints(zmq::socket_t* readrep_sock, std::map<std::s
           tmp=ip + ":" + store_port;
           tmp="tcp://"+ tmp;
           try {
+            mm_mtx.lock();
             mm_sock->connect(tmp.c_str());
+            mm_mtx.unlock();
             std::cout<<"MMUtilities::ConnectToEndpoints new connection to middleman "<<tmp<<std::endl;
             ++num_new_connections;
           } catch(zmq::error_t& err){
@@ -138,12 +146,12 @@ int MMUtilities::ConnectToEndpoints(zmq::socket_t* readrep_sock, std::map<std::s
     } // end loop over services in broadcast
     
     // prune inactive endpoints
+    /*
     std::vector<std::map<std::string,Store*>::iterator> to_erase;
     for(std::map<std::string,Store*>::iterator it=readrep_conns.begin(); it!=readrep_conns.end(); ++it){
       if(active_endpoints.count(it->first)==0){
         std::cout<<"Booting inactive endpoint "<<it->first<<std::endl;
-        /*
-        // could explicitly disconnect?
+        // explicitly disconnect too
         std::string store_port="";
         it->second->Get("remote_port",store_port);
         std::string tmp="tcp://"+ it->first + ":" + store_port;
@@ -152,8 +160,9 @@ int MMUtilities::ConnectToEndpoints(zmq::socket_t* readrep_sock, std::map<std::s
           write_sock->disconnect(tmp.c_str());
         } catch(zmq::error_t& err){
           std::cerr<<"MMUtilities::ConnectToEndpoints error disconnecting from stale socket "<<tmp<<": "<<err.what()<<std::endl;
+          // this returns "no such file or directory"?? does it make sense to call disconnect on a sub socket?
+          // we should put the two into separate loops
         }
-        */
         delete it->second;
         to_erase.push_back(it);
       }
@@ -161,8 +170,28 @@ int MMUtilities::ConnectToEndpoints(zmq::socket_t* readrep_sock, std::map<std::s
     for(std::map<std::string,Store*>::iterator it : to_erase){
       readrep_conns.erase(it);
     }
+    */
     
     return num_new_connections;
 }
 
-
+bool MMUtilities::ClearConnections(zmq::socket_t* sock, std::map<std::string,Store*> &conns, std::mutex& sock_mtx){
+    // prune and disconnect from all endpoints
+    bool ok=true;
+    for(std::map<std::string,Store*>::iterator it=conns.begin(); it!=conns.end(); ++it){
+      std::string store_port="";
+      it->second->Get("remote_port",store_port);
+      std::string tmp="tcp://"+ it->first + ":" + store_port;
+      try {
+        sock_mtx.lock();
+        sock->disconnect(tmp.c_str());
+        sock_mtx.unlock();
+      } catch(zmq::error_t& err){
+        std::cerr<<"MMUtilities::ClearConnections error disconnecting from client "<<tmp<<": "<<err.what()<<std::endl;
+        ok = false;
+      }
+      delete it->second;
+    }
+    conns.clear();
+    return ok;
+}
diff --git a/MMUtilities.h b/MMUtilities.h
index d9d3bd7..37638fd 100644
--- a/MMUtilities.h
+++ b/MMUtilities.h
@@ -4,6 +4,7 @@
 #include <DAQUtilities.h>
 #include <errno.h>     // for errno
 #include <set>
+#include <mutex>
 
 using namespace ToolFramework;
 
@@ -13,7 +14,9 @@ class MMUtilities : public DAQUtilities {
   
   MMUtilities(zmq::context_t* zmqcontext);
 
-  int ConnectToEndpoints(zmq::socket_t* readrep_sock, std::map<std::string,Store*> &readrep_conns, int read_port_num, zmq::socket_t* write_sock, std::map<std::string,Store*> &write_conns, int write_port_num, zmq::socket_t* mm_sock, std::map<std::string, Store*> &mm_conns, int mm_port_num); ///< Add to standard ports assumed to be associated with all found services.
+  int ConnectToEndpoints(zmq::socket_t* readrep_sock, std::map<std::string,Store*> &readrep_conns, int read_port_num, std::mutex& readrep_mtx, zmq::socket_t* write_sock, std::map<std::string,Store*> &write_conns, int write_port_num, std::mutex& write_mtx, zmq::socket_t* mm_sock, std::map<std::string, Store*> &mm_conns, int mm_port_num, std::mutex& mm_mtx); ///< Add to standard ports assumed to be associated with all found services.
+
+  bool ClearConnections(zmq::socket_t* sock, std::map<std::string,Store*> &conns, std::mutex& sock_mtx);
 
 private:
 
diff --git a/Postgres.cpp b/Postgres.cpp
index 89be8cc..eff8a74 100755
--- a/Postgres.cpp
+++ b/Postgres.cpp
@@ -13,17 +13,8 @@ void Postgres::SetVerbosity(int verb){
 // TODO probably could do better with the exception handling throughout this file
 pqxx::connection* Postgres::OpenConnection(std::string* err){
 	if(verbosity>v_debug) std::cout<<"Opening Connection"<<std::endl;
+	
 	try{
-		// if we already have a connection open, nothing to do
-		if(conn!=nullptr && conn->is_open()){
-			if(verbosity>v_debug) std::cout<<"Connection already open"<<std::endl;
-			return conn;
-		} else if(conn){
-			// conn not null, but is not open...
-			delete conn; // should we do this? e.g. lazy connections will open only on first use.
-		}
-		
-		// otherwise form the connection string
 		std::stringstream tmp;
 		if(dbname!="")   tmp<<" dbname="<<dbname;
 		if(port!=-1)     tmp<<" port="<<port;
@@ -34,11 +25,12 @@ pqxx::connection* Postgres::OpenConnection(std::string* err){
 		
 		// attempt to connect to the database
 		if(verbosity>v_debug) std::cout<<"connecting with string '"<<tmp.str()<<"'"<<std::endl;
-		conn = new pqxx::connection(tmp.str().c_str());
+		pqxx::connection* conn = new pqxx::connection(tmp.str().c_str());
 		
 		// verify we succeeded
 		// "don't use is_open(), use the broken_connection exception", they say. Hmm.
 		// but will that be thrown now, or only when we try to *use* the connection, for a transaction?
+		// may depend on the connection type... let's just check?
 		if(!conn->is_open()){
 			std::cerr<<"Failed to connect to the database! Connection string was: '"
 			         <<tmp.str()<<"', please verify connection details"<<std::endl;
@@ -46,8 +38,7 @@ pqxx::connection* Postgres::OpenConnection(std::string* err){
 			return nullptr;
 		}
 		return conn;
-	}
-	catch (const pqxx::broken_connection &e){
+	} catch (const pqxx::broken_connection &e){
 		// as usual the doxygen sucks, but it seems this doesn't provide
 		// any further methods to obtain information about the failure mode,
 		// so probably not useful to catch this explicitly.
@@ -61,7 +52,48 @@ pqxx::connection* Postgres::OpenConnection(std::string* err){
 	return nullptr;
 }
 
-bool Postgres::CloseConnection(std::string* err){
+bool Postgres::SetConnection(pqxx::connection* conn){
+	if(!CheckConnection(conn)) return false;
+	if(m_conn!=nullptr && CheckConnection(m_conn)){
+		CloseConnection(m_conn);
+	}
+	m_conn = conn;
+	return true;
+}
+
+bool Postgres::CheckConnection(pqxx::connection*& conn, std::string* err){
+	// if no connection given, use the member one.
+	// this means users do not need to hold onto connection separately, but must be single-threaded.
+	if(conn==nullptr && m_conn!=nullptr) conn=m_conn;
+	if(conn==nullptr){
+		std::cerr<<"CheckConnection with no connection!"<<std::endl;
+		if(err) *err = "CheckConnection with no connection!";
+	} else {
+		// otherwise check it
+		try{
+			if(conn->is_open()){
+				if(verbosity>v_debug) std::cout<<"Connection already open"<<std::endl;
+				return true;
+			} else {
+				if(err) *err = "CheckConnection found Connection closed!";
+				std::cerr<<"CheckConnection found Connection closed!"<<std::endl;
+			}
+		} catch (const pqxx::broken_connection &e){
+			// as usual the doxygen sucks, but it seems this doesn't provide
+			// any further methods to obtain information about the failure mode,
+			// so probably not useful to catch this explicitly.
+			std::cerr << e.what() << std::endl;
+			if(err) *err = e.what();
+		}
+		catch (std::exception const &e){
+			std::cerr << e.what() << std::endl;
+			if(err) *err = e.what();
+		}
+	}
+	return false;
+}
+
+bool Postgres::CloseConnection(pqxx::connection* conn, std::string* err){
 	if(verbosity>v_debug){
 		std::cout<<"Closing connection"<<std::endl;
 	}
@@ -94,8 +126,11 @@ bool Postgres::CloseConnection(std::string* err){
 }
 
 Postgres::~Postgres(){
-	CloseConnection();
-	if(conn) delete conn;
+	if(m_conn){
+		CloseConnection(m_conn);
+		delete m_conn;
+		m_conn=nullptr;
+	}
 }
 
 Postgres::Postgres(){}
@@ -117,12 +152,12 @@ void Postgres::Init(std::string hostname_in, std::string hostip_in, int port_in,
 }
 
 // XXX reminder that pqxx::result is a reference-counting wrapper and is not thread-safe! XXX
-bool Postgres::Query(std::string query, int nret, pqxx::result* res, pqxx::row* row, std::string* err){
+bool Postgres::Query(pqxx::connection* conn, std::string query, int nret, pqxx::result* res, pqxx::row* row, std::string* err){
 	
-	// maybe this is redundant since OpenConnection will check is_open (against recommendations)
+	// maybe this is redundant since CheckConnection will check is_open (against recommendations)
 	for(int tries=0; tries<2; ++tries){
 		// ensure we have a connection to work with
-		if(OpenConnection(err)==nullptr){
+		if(!CheckConnection(conn, err)){
 			// no connection to batabase -> abort
 			return false;
 		}
@@ -175,8 +210,9 @@ bool Postgres::Query(std::string query, int nret, pqxx::result* res, pqxx::row*
 		catch (const pqxx::broken_connection &e){
 			// if our connection is broken after all, disconnect, reconnect and retry
 			if(tries==0){
-				CloseConnection();
-				delete conn; conn=nullptr;
+				CloseConnection(conn);
+				delete conn;
+				conn = OpenConnection();
 				continue;
 			} else {
 				std::cerr<<"Postgres::Query error - broken connection, failed to re-establish it"<<std::endl;
@@ -208,12 +244,12 @@ bool Postgres::Query(std::string query, int nret, pqxx::result* res, pqxx::row*
 	return false;
 }
 
-bool Postgres::QueryAsStrings(std::string query, std::vector<std::string> *results, char row_or_col, std::string* err){
+bool Postgres::QueryAsStrings(pqxx::connection* conn, std::string query, std::vector<std::string> *results, char row_or_col, std::string* err){
 	// generically run a query, without knowing how many returns are expected.
 	// we'll need to get the results in a generic pqxx::result, and specify the number
 	// of returned rows is >1. If there's fewer, it'll just return an empty container.
 	pqxx::result res;
-	get_ok = Query(query, 2, &res, nullptr, err);
+	get_ok = Query(conn, query, 2, &res, nullptr, err);
 	// if the query failed, the user didn't provide means for a return, or the query had no return,
 	// then we have no need to parse the response and we're done.
 	if(not get_ok || results==nullptr || res.size()==0) return get_ok;
@@ -237,13 +273,13 @@ bool Postgres::QueryAsStrings(std::string query, std::vector<std::string> *resul
 	return true;
 }
 
-bool Postgres::QueryAsJsons(std::string query, std::vector<std::string> *results, std::string* err){
+bool Postgres::QueryAsJsons(pqxx::connection* conn, std::string query, std::vector<std::string> *results, std::string* err){
 	// generically run a query, without knowing how many returns are expected.
 	// we'll need to get the results in a generic pqxx::result, and specify the number
 	// of returned rows is >1. If there's fewer, it'll just return an empty container.
 	//printf("QueryAsJsons running '%s'\n",query.c_str());
 	pqxx::result res;
-	get_ok = Query(query, 2, &res, nullptr, err);
+	get_ok = Query(conn, query, 2, &res, nullptr, err);
 	// if the query failed, the user didn't provide means for a return, or the query had no return,
 	// then we have no need to parse the response and we're done.
 	if(not get_ok || results==nullptr || res.size()==0) return get_ok;
@@ -258,8 +294,7 @@ bool Postgres::QueryAsJsons(std::string query, std::vector<std::string> *results
 			// but to convert this into JSON, strings should be quoted:
 			// i.e. { "field1":3, "field2":"cat", "field3":{"iam":"ajson"} }
 			// this means we need to add enclosing quotes *only* for string fields
-			if((it->type()==18) || (it->type()==25) ||
-                           (it->type()==1042) || (it->type()==1043)){
+			if((it->type()==18) || (it->type()==25) || (it->type()==1042) || (it->type()==1043)){
 				tmpval = "\""+tmpval+"\"";
 			}
 			tmp << "\"" << it->name() << "\":"<< tmpval;
@@ -272,14 +307,14 @@ bool Postgres::QueryAsJsons(std::string query, std::vector<std::string> *results
 	return true;
 }
 
-bool Postgres::Promote(int wait_seconds, std::string* err){
+bool Postgres::Promote(pqxx::connection* conn, int wait_seconds, std::string* err){
 	std::string promote_query = "pg_promote(TRUE,"+std::to_string(wait_seconds)+")";
 	// TRUE says to wait; otherwise we don't know whether the promotion succeeded.
 	// wait is the time we wait for the promotion to succeed before aborting.
-	return Query(promote_query, 0, nullptr, nullptr, err);
+	return Query(conn, promote_query, 0, nullptr, nullptr, err);
 }
 
-bool Postgres::Demote(int wait_seconds, std::string* err){
+bool Postgres::Demote(pqxx::connection* conn, int wait_seconds, std::string* err){
 	/* FIXME implement using repmgr or pgbackrest */
 	// as a minimal start, we can just make the standby.signal file and issue `pg_ctl restart`
 	// however, if there are inconsistencies in this instance and the new master, startup may fail!
@@ -288,7 +323,7 @@ bool Postgres::Demote(int wait_seconds, std::string* err){
 //	// get db name
 //	std::string query_string = "SELECT current_database()";
 //	std::string dbname;
-//	get_ok = ExecuteQuery(query_string, dbname);
+//	get_ok = ExecuteQuery(conn, query_string, dbname);
 //	if(not get_ok){
 //		std::string msg = "Failed to get name of current database in Postgres::Demote!";
 //		std::cerr<<msg<<std::endl;
@@ -299,7 +334,7 @@ bool Postgres::Demote(int wait_seconds, std::string* err){
 	// and db directory
 	std::string query_string = "SELECT setting FROM pg_settings WHERE name='data_directory'";
 	std::string dbdir;
-	get_ok = ExecuteQuery(query_string, dbdir);
+	get_ok = ExecuteQuery(conn, query_string, dbdir);
 	
 	// make the signal file that the db should start up in standby mode
 	std::string flagfilepath = dbdir + "/standby.signal";
@@ -383,7 +418,7 @@ bool Postgres::Demote(int wait_seconds, std::string* err){
 	// since some of these are a little ambiguous we should double check the server status explicitly
 	// let's do a sanity check to see if we can query the database, and if we're now in recovery mode
 	bool in_recovery=false;
-	get_ok = ExecuteQuery("SELECT pg_is_in_recovery()",in_recovery);
+	get_ok = ExecuteQuery(conn, "SELECT pg_is_in_recovery()",in_recovery);
 	if(get_ok && in_recovery){
 		// all looks good!
 		std::cout<<"Database restarted in standby mode"<<std::endl;
@@ -428,9 +463,9 @@ bool Postgres::Demote(int wait_seconds, std::string* err){
 // Quoting functions
 // =================
 // quote field or table names (nominally use double quotes)
-bool Postgres::pqxx_quote_name(const std::string& in, std::string& out, std::string* err){
+bool Postgres::pqxx_quote_name(pqxx::connection* conn, const std::string& in, std::string& out, std::string* err){
 	out = in; // often this will be sufficient
-	if(OpenConnection(err)==nullptr) return false;
+	if(!CheckConnection(conn, err)) return false;
 	try {
 		out = conn->quote_name(in);
 		return true;
@@ -445,12 +480,12 @@ bool Postgres::pqxx_quote_name(const std::string& in, std::string& out, std::str
 
 // quote values (nominally, use single quotes)
 // TODO error signalling here sucks; shouldn't just return empty on failure!
-bool Postgres::pqxx_quote(const std::string& in, std::string& out, std::string* err){
+bool Postgres::pqxx_quote(pqxx::connection* conn, const std::string& in, std::string& out, std::string* err){
 	out = in;
 	//return pqxx::nullconnection{}.quote(string);
 	// annoyingly we can't use a null connection just to get libpqxx to quote things for us;
 	// we must have a valid connection to a real database :/
-	if(OpenConnection(err)==nullptr) return false;
+	if(!CheckConnection(conn, err)) return false;
 	try {
 		out = conn->quote(in);
 		return true;
diff --git a/Postgres.h b/Postgres.h
index 61d73a9..20a4b4f 100755
--- a/Postgres.h
+++ b/Postgres.h
@@ -22,23 +22,28 @@ class Postgres {
 	// open a connection to the database
 	pqxx::connection* OpenConnection(std::string* err=nullptr);
 	// close the connection to the database. returns success, for whatever failure implies.
-	bool CloseConnection(std::string* err=nullptr);
+	bool CloseConnection(pqxx::connection* conn, std::string* err=nullptr);
+	// check a connection to the database is open. if passed pointer is null,
+	// will try to fall back to internal connection m_conn
+	bool CheckConnection(pqxx::connection*& conn, std::string* err=nullptr);
+	// update internal connection
+	bool SetConnection(pqxx::connection* conn);
 	
 	// wrapper around exec since we handle the transaction and connection.
 	// nret specifies the expected number of returned rows from the query.
 	// res and row are outputs. return value is success.
-	bool Query(std::string query, int nret=0, pqxx::result* res=nullptr, pqxx::row* row=nullptr, std::string* err=nullptr);
+	bool Query(pqxx::connection* conn, std::string query, int nret=0, pqxx::result* res=nullptr, pqxx::row* row=nullptr, std::string* err=nullptr);
 	
 	// one that returns the value of many fields from one row as strings in a vector (row_or_col='r')
 	// or the value of one field from many rows (row_or_col='c')
-	bool QueryAsStrings(std::string query, std::vector<std::string> *results, char row_or_col, std::string* err=nullptr);
+	bool QueryAsStrings(pqxx::connection* conn, std::string query, std::vector<std::string> *results, char row_or_col, std::string* err=nullptr);
 	
 	// one that returns rows as json strings representing maps of fieldname:value
 	// we uhhhhh need to check this works ok with nested strings and stuff
-	bool QueryAsJsons(std::string query, std::vector<std::string> *results, std::string* err=nullptr);
+	bool QueryAsJsons(pqxx::connection* conn, std::string query, std::vector<std::string> *results, std::string* err=nullptr);
 	
-	bool Promote(int wait_seconds=60, std::string* err=nullptr);
-	bool Demote(int wait_seconds=60, std::string* err=nullptr);
+	bool Promote(pqxx::connection* conn, int wait_seconds=60, std::string* err=nullptr);
+	bool Demote(pqxx::connection* conn, int wait_seconds=60, std::string* err=nullptr);
 	
 	private:
 	int verbosity=1;
@@ -48,7 +53,7 @@ class Postgres {
 	int v_debug=3;
 	std::string logmessage;
 	int get_ok;
-	pqxx::connection* conn=nullptr;
+	pqxx::connection* m_conn=nullptr;
 	
 	// default connection details
 	std::string dbname="";
@@ -73,12 +78,12 @@ class Postgres {
 //	}
 	
 	template <typename... Ts>
-	bool ExecuteQuery(std::string query_string, Ts&&... rets){
+	bool ExecuteQuery(pqxx::connection* conn, std::string query_string, Ts&&... rets){
 		// run an SQL query and try to pass the results
 		// into a parameter pack. the passed arguments
 		// must be compatible with the returned columns
 		pqxx::row local_row;
-		bool success = Query(query_string, 1, nullptr, &local_row);
+		bool success = Query(conn, query_string, 1, nullptr, &local_row);
 		if(not success) return false; // query failed
 		
 		success = ExpandRow<sizeof...(Ts), Ts&&...>::expand(local_row, std::forward<Ts>(rets)...);
@@ -162,11 +167,11 @@ class Postgres {
 	////////
 	// helper function for insertions
 	template <typename... Rest>
-	bool Insert(std::string tablename, std::vector<std::string> &fields, std::string* err, Rest... args){
-		// maybe this is redundant since OpenConnection will check is_open (against recommendations)
+	bool Insert(pqxx::connection*& conn, std::string tablename, std::vector<std::string> &fields, std::string* err, Rest... args){
+		// maybe this is redundant since CheckConnection will check is_open (against recommendations)
 		for(int tries=0; tries<2; ++tries){
 			// ensure we have a connection to work with
-			if(OpenConnection(err)==nullptr){
+			if(CheckConnection(conn, err)){
 				// no connection to batabase -> abort
 				return false;
 			}
@@ -194,8 +199,10 @@ class Postgres {
 			catch (const pqxx::broken_connection &e){
 				// if our connection is broken after all, disconnect, reconnect and retry
 				if(tries==0){
-					CloseConnection();
-					delete conn; conn=nullptr;
+					CloseConnection(conn);
+					delete conn;
+					conn=nullptr;
+					conn=OpenConnection();
 					continue;
 				} else {
 					std::cerr<<"Postgres::Query error - broken connection, failed to re-establish it"<<std::endl;
@@ -232,10 +239,10 @@ class Postgres {
 	// https://libpqxx.readthedocs.io/en/6.3/a00255.html#ga81fe65fbb9561af7c5f0b33a9fe27e5a
 	// -------
 	// quote field or table names (nominally use double quotes)
-	bool pqxx_quote_name(const std::string& in, std::string& out, std::string* err=nullptr);
+	bool pqxx_quote_name(pqxx::connection* conn, const std::string& in, std::string& out, std::string* err=nullptr);
 	
 	// quote values (nominally, use single quotes)
-	bool pqxx_quote(const std::string& in, std::string& out, std::string* err=nullptr);
+	bool pqxx_quote(pqxx::connection* conn, const std::string& in, std::string& out, std::string* err=nullptr);
 	// ------------------ //
 	
 };
diff --git a/Query.cpp b/Query.cpp
index 7949e1c..d5888ab 100755
--- a/Query.cpp
+++ b/Query.cpp
@@ -1,11 +1,12 @@
 #include "Query.h"
 
 // constructor from elements, given a direct SQL query string
-Query::Query(zmq::message_t& client_id_in, zmq::message_t& msg_id_in, const std::string& database_in, const std::string& query_in, uint32_t query_ok_in, std::string response_in){
+Query::Query(zmq::message_t& client_id_in, zmq::message_t& msg_id_in, const std::string& database_in, const std::string& topic_in, const std::string& query_in, uint32_t query_ok_in, std::string response_in){
 	client_id.move(&client_id_in);
 	message_id.move(&msg_id_in);
 	
 	database = database_in.substr(0,database_in.find('\0'));
+	topic = topic_in.substr(0,topic_in.find('\0'));
 	query = query_in.substr(0,query_in.find('\0'));
 	
 	//std::cout<<"building query for db: '"<<database<<"', query_string: '"<<query<<"'"<<std::endl;
@@ -22,6 +23,7 @@ Query::Query(const Query& in){
 	client_id.copy(&in.client_id);
 	message_id.copy(&in.message_id);
 	database = in.database;
+	topic = in.topic;
 	query = in.query;
 	query_ok = in.query_ok;
 	response = in.response;
diff --git a/Query.h b/Query.h
index 13cd471..894cb00 100755
--- a/Query.h
+++ b/Query.h
@@ -6,7 +6,7 @@
 #include <string>
 
 struct Query{
-	Query(zmq::message_t& client_id_in, zmq::message_t& msg_id_in, const std::string& database_in, const std::string& query_in, uint32_t query_ok_in=0, std::string response_in="NULL");
+	Query(zmq::message_t& client_id_in, zmq::message_t& msg_id_in, const std::string& database_in, const std::string& topic_in, const std::string& query_in, uint32_t query_ok_in=0, std::string response_in="NULL");
 	Query(const Query& in);
 	void Print();
 	
@@ -14,6 +14,7 @@ struct Query{
 	zmq::message_t message_id;
 	std::string database;
 	std::string query;
+	std::string topic;
 	uint32_t query_ok;
 	std::vector<std::string> response;
 	int retries;
diff --git a/ReceiveSQL.cpp b/ReceiveSQL.cpp
index 2582745..be5c6f1 100755
--- a/ReceiveSQL.cpp
+++ b/ReceiveSQL.cpp
@@ -61,11 +61,13 @@ bool ReceiveSQL::Execute(){
 	auto last = std::chrono::high_resolution_clock::now();
 	
 	// find new clients
+	/* now handled by background thread
 	Log("Finding new clients",20);
 	get_ok = FindNewClients_v2();
 	timers["Finding Clients"] =
 	               std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - last).count();
 	last = std::chrono::high_resolution_clock::now();
+	*/
 	
 	// poll the input sockets for messages
 	Log("Polling input sockets",20);
@@ -89,11 +91,11 @@ bool ReceiveSQL::Execute(){
 		if(in_polls.at(i).revents & ZMQ_POLLIN){
 			switch(i){
 				case 0: {
-					//pollsmsg+="M";  too many...
+					pollsmsg+="R";
 					break;
 				}
 				case 1: {
-					pollsmsg+="R";
+					//pollsmsg+="MM"; // middleman check-ins, don't care
 					break;
 				}
 				case 2: {
@@ -125,6 +127,8 @@ bool ReceiveSQL::Execute(){
 	               std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - last).count();
 	last = std::chrono::high_resolution_clock::now();
 	
+	/*
+	// moved to thread
 	if(am_master){
 		Log("Getting Client Multicast Messages",20);
 		get_ok = GetMulticastMessages();
@@ -132,6 +136,7 @@ bool ReceiveSQL::Execute(){
 	timers["Get Multicasts"] =
 	               std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - last).count();
 	last = std::chrono::high_resolution_clock::now();
+	*/
 	
 	Log("Getting Middleman Checkin",20);
 	get_ok = GetMiddlemanCheckin();
@@ -159,6 +164,8 @@ bool ReceiveSQL::Execute(){
 	               std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - last).count();
 	last = std::chrono::high_resolution_clock::now();
 	
+	/*
+	// now handled by ReceiveSQL::MulticastWorker, which gets invoked by Utilities and does not need to be called explicitly
 	if(am_master){
 		Log("Running Next Fire-and-Forget Message",20);
 		get_ok = RunNextMulticastMsg();
@@ -166,6 +173,7 @@ bool ReceiveSQL::Execute(){
 	timers["Run Multicast"] =
 	               std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - last).count();
 	last = std::chrono::high_resolution_clock::now();
+	*/
 	
 	// poll the output sockets for listeners
 	Log("Polling output sockets",20);
@@ -208,8 +216,6 @@ bool ReceiveSQL::Execute(){
 	get_ok = TrimQueue("rd_txn_queue");
 	Log("Trimming Ack Queue",20);
 	get_ok = TrimQueue("response_queue");
-	Log("Trimming In Multicast Deque",20);
-	get_ok = TrimDequeue("in_multicast_queue");
 	Log("Trimming Out Logging Deque",20);
 	get_ok = TrimDequeue("out_multicast_queue");
 	Log("Trimming Cache",20);
@@ -239,7 +245,7 @@ bool ReceiveSQL::Execute(){
 	
 	// tracking loop rate
 	int loop_ms = std::chrono::duration_cast<std::chrono::milliseconds>(loop_end - loop_start).count();
-	if(loop_ms>3000){
+	if(loop_ms>100){
 		Log("Warning: Middleman Execute took "+std::to_string(loop_ms)+"ms!",v_warning);
 		std::string msg="Timers were: ";
 		for(auto&& atimer : timers){
@@ -262,8 +268,34 @@ bool ReceiveSQL::Finalise(){
 	Log("Removing Discoverable Services",3);
 	if(utilities) utilities->RemoveService("middleman");
 	
-	Log("Closing multicast socket",3);
-	close(multicast_socket);
+	// signal background thread connecting to new clients to stop
+	m_util.KillThread(findclients_args);
+	
+	// signal background receiver thread to stop
+	multicast_listener_args->running=false;
+	m_util.KillThread(multicast_listener_args);
+	
+	// signal background workers to stop
+	for(MulticastWorker_args* thread_args : multicast_worker_args){
+		thread_args->running=false;
+	}
+	// wait for it to finish processing any multicast queries in its buffer
+	// it'll close the socket and tell us when it's done
+	int secs_to_wait=60;
+	for(int i=0; i<secs_to_wait; ++i){
+		bool all_done=true;
+		for(MulticastWorker_args* thread_args : multicast_worker_args){
+			if(!thread_args->finished) all_done=false;
+		}
+		if(all_done) break;
+		std::this_thread::sleep_for(std::chrono::seconds(1));
+	}
+	for(MulticastWorker_args* thread_args : multicast_worker_args){
+		m_util.KillThread(thread_args);
+		delete thread_args;
+		thread_args=0;
+	}
+	multicast_worker_args.clear();
 	
 	Log("Deleting Utilities",3);
 	if(utilities){ delete utilities; utilities=nullptr; }
@@ -271,11 +303,6 @@ bool ReceiveSQL::Finalise(){
 	Log("Deleting ServiceDiscovery",3);
 	if(service_discovery){ delete service_discovery; service_discovery=nullptr; }
 	
-	Log("Clearing known connections",3);
-	clt_rtr_connections.clear();
-	mm_rcv_connections.clear();
-	clt_sub_connections.clear();
-	
 	// delete sockets
 	Log("Deleting sockets",3);
 	if(clt_sub_socket){ delete clt_sub_socket; clt_sub_socket=nullptr; }
@@ -344,11 +371,13 @@ bool ReceiveSQL::InitPostgres(Store& m_variables, const std::string& dbname){
 	                dbpasswd,
 	                dbname);
 	
-	// try to open a connection to ensure we can do, or else bail out now.
-	if(!m_database.OpenConnection()){
+	// try to open a connection to ensure we can do, or else bail out now
+	pqxx::connection* conn = m_database.OpenConnection();
+	if(conn==nullptr){
 		Log(Concat("Error! Failed to open connection to the ",dbname," database!"),0);
 		return false;
 	}
+	m_database.SetConnection(conn); // yeah this seems circular, urgh.
 	
 	return true;
 }
@@ -366,6 +395,8 @@ bool ReceiveSQL::InitMulticast(Store& m_variables){
 	// FIXME add to config file
 	m_variables.Get("multicast_port",multicast_port);
 	m_variables.Get("multicast_address",multicast_address);
+	int num_multicast_threads=10;
+	m_variables.Get("num_multicast_threads",num_multicast_threads);
 	
 	// set up multicast socket for sending logging & monitoring data
 	multicast_socket = socket(AF_INET, SOCK_DGRAM, 0);
@@ -379,7 +410,7 @@ bool ReceiveSQL::InitMulticast(Store& m_variables){
 	l.l_onoff  = 0;  // whether to linger
 	l.l_linger = 0;  // seconds to linger for
 	get_ok = setsockopt(multicast_socket, SOL_SOCKET, SO_LINGER, (char*) &l, sizeof(l));
-	int a =1;
+	int a = 1;
 	setsockopt(multicast_socket, SOL_SOCKET, SO_REUSEADDR, &a, sizeof(a));
 	if(get_ok!=0){
 		Log(std::string{"Failed to set multicast socket linger with error "}+strerror(errno),v_error);
@@ -389,6 +420,7 @@ bool ReceiveSQL::InitMulticast(Store& m_variables){
 	// set the socket to non-blocking mode - seems like a good idea...? XXX
 	get_ok = fcntl(multicast_socket, F_SETFL, O_NONBLOCK);
 	if(get_ok!=0){
+		// maybe this is not essential..? XXX
 		Log(std::string{"Failed to set multicast socket to non-blocking with error "}+strerror(errno),v_error);
 		//return false;
 	}
@@ -435,9 +467,35 @@ bool ReceiveSQL::InitMulticast(Store& m_variables){
 		return false;
 	}
 	
+	multicast_listener_args = new MulticastReceive_args;
+	multicast_listener_args->parent = this;
+	multicast_listener_args->finished = false;
+	multicast_listener_args->multicast_socket = multicast_socket;
+	multicast_listener_args->multicast_addr = &multicast_addr;
+	multicast_listener_args->multicast_addrlen = multicast_addrlen;
 	// we can poll with zmq ...
-	in_polls.emplace_back(zmq::pollitem_t{NULL, multicast_socket, ZMQ_POLLIN, 0});
-	out_polls.emplace_back(zmq::pollitem_t{NULL, multicast_socket, ZMQ_POLLOUT, 0});
+	multicast_listener_args->poll = zmq::pollitem_t{NULL, multicast_socket, ZMQ_POLLIN, 0};
+	m_util.CreateThread("multicast_receiver", &GetMulticastMessages, multicast_listener_args);
+	
+	// the majority of our messages come via multicast, and each may be quite large,
+	// with the corresponding queries taking a while (order seconds).
+	// We need to make sure these don't delay responses on Read/Write sockets
+	// which have a timeout, so let's handle multicasts in a separate thread
+	for(int i=0; i<num_multicast_threads; ++i){
+		MulticastWorker_args* thread_args=new MulticastWorker_args();
+		thread_args->parent = this;
+		thread_args->running=true;
+		thread_args->finished=false;
+		thread_args->conn = m_databases.at("daq").OpenConnection();
+		if(thread_args->conn==nullptr){
+			Log("Failed to open connection to database for multicast thread!",v_error);
+			return false;
+		}
+		thread_args->last_log_insert = boost::posix_time::microsec_clock::universal_time();
+		thread_args->last_mon_insert = boost::posix_time::microsec_clock::universal_time();
+		m_util.CreateThread("multicast_worker", &MulticastWorker, thread_args);
+		multicast_worker_args.push_back(thread_args);
+	}
 	
 	return true;
 }
@@ -532,6 +590,8 @@ bool ReceiveSQL::InitZMQ(Store& m_variables){
 		// don't linger too long, it looks like the program crashed.
 		clt_sub_socket->setsockopt(ZMQ_LINGER, 10);
 		clt_sub_socket->setsockopt(ZMQ_SUBSCRIBE,"",0);
+		clt_sub_socket->setsockopt(ZMQ_RCVHWM,10000);
+		clt_sub_socket->setsockopt(ZMQ_BACKLOG,1000);
 		// we will connect this socket to clients with the utilities class
 	}
 	
@@ -540,6 +600,8 @@ bool ReceiveSQL::InitZMQ(Store& m_variables){
 	clt_rtr_socket = new zmq::socket_t(*context, ZMQ_ROUTER);
 	clt_rtr_socket->setsockopt(ZMQ_SNDTIMEO, clt_rtr_socket_timeout);
 	clt_rtr_socket->setsockopt(ZMQ_RCVTIMEO, clt_rtr_socket_timeout);
+	clt_rtr_socket->setsockopt(ZMQ_RCVHWM,10000);
+	clt_rtr_socket->setsockopt(ZMQ_BACKLOG,1000);
 	// make router transfer connections with an already seen ZMQ_IDENTITY to a new connection
 	// rather than rejecting the new connection attempt.
 	/*
@@ -676,6 +738,14 @@ bool ReceiveSQL::InitServiceDiscovery(Store& m_variables){
 	
 	// note that it is not necessary to register the RemoteControl service,
 	// this is automatically done by the ServiceDiscovery class.
+	
+	// this will be done in a bg thread
+	findclients_args = new FindClients_args;
+	findclients_args->parent = this;
+	findclients_args->utilities = utilities;
+	
+	m_util.CreateThread("findclients", &FindNewClients_v2, findclients_args);
+	
 	return true;
 }
 
@@ -713,6 +783,9 @@ bool ReceiveSQL::InitControls(Store& m_variables){
 	SC_vars.Add("Clients",SlowControlElementType(INFO));
 	SC_vars["Clients"]->SetValue("None");
 	
+	SC_vars.Add("ClearConnections",SlowControlElementType(BUTTON));
+	SC_vars["ClearConnections"]->SetValue(false);
+	
 	// the SlowControlCollection thread listens for requests to get or set the registered controls,
 	// and responds with or updates its internal control values. Note that (for now) it does not
 	// take any action to effect the state of things outside it, so we will need to manually keep
@@ -804,6 +877,7 @@ bool ReceiveSQL::InitMessaging(Store& m_variables){
 
 // ««-------------- ≪ °◇◆◇° ≫ --------------»»
 
+/*
 bool ReceiveSQL::FindNewClients(){
 	
 	int new_connections=0;
@@ -830,37 +904,36 @@ bool ReceiveSQL::FindNewClients(){
 		Log("No new clients found",21);
 	}
 	
-	/* needs fixing to uncomment
-	std::cout<<"We have: "<<connections.size()<<" connected clients"<<std::endl;
-	std::cout<<"Connections are: "<<std::endl;
-	for(auto&& athing : connections){
-		std::string service;
-		athing.second->Get("msg_value",service);
-		std::cout<<service<<" connected on "<<athing.first<<std::endl;
-	}
-	*/
-	
 	return true;
 }
+*/
 
 // ««-------------- ≪ °◇◆◇° ≫ --------------»»
 
-bool ReceiveSQL::FindNewClients_v2(){
+void ReceiveSQL::FindNewClients_v2(Thread_args* arg){
+	
+	// check for new clients
+	FindClients_args* m_args=reinterpret_cast<FindClients_args*>(arg);
+	ReceiveSQL& p = *m_args->parent;
 	
-	int clt_rtr_conns = clt_rtr_connections.size();
-	int clt_sub_conns = clt_sub_connections.size();
-	int mm_conns = mm_rcv_connections.size();
+	int clt_rtr_conns = m_args->clt_rtr_connections.size();
+	int clt_sub_conns = m_args->clt_sub_connections.size();
+	int mm_conns = m_args->mm_rcv_connections.size();
 	
-	int new_connections = utilities->ConnectToEndpoints(clt_rtr_socket, clt_rtr_connections, clt_rtr_port, clt_sub_socket, clt_sub_connections, clt_sub_port, mm_rcv_socket, mm_rcv_connections, mm_rcv_port);
+	p.Log("checking for new clients",22);
+	int new_connections = m_args->utilities->ConnectToEndpoints(p.clt_rtr_socket, m_args->clt_rtr_connections, p.clt_rtr_port, p.clt_rtr_mtx,
+	                                                            p.clt_sub_socket, m_args->clt_sub_connections, p.clt_sub_port, p.clt_sub_mtx,
+	                                                            p.mm_rcv_socket, m_args->mm_rcv_connections, p.mm_rcv_port, p.mm_rcv_mtx);
 	
 	if(new_connections>0){
-		Log("Made "+std::to_string(new_connections)+" new connections!",2);
-		Log("Made "+std::to_string(clt_rtr_connections.size()-clt_rtr_conns)
+		p.Log("Made "+std::to_string(new_connections)+" new connections!",2);
+		p.Log("Made "+std::to_string(m_args->clt_rtr_connections.size()-clt_rtr_conns)
 		   +" new read/reply socket connections",3);
-		Log("Made "+std::to_string(clt_sub_connections.size()-clt_sub_conns)
+		p.Log("Made "+std::to_string(m_args->clt_sub_connections.size()-clt_sub_conns)
 		   +" new write socket connections",3);
-		Log("Made "+std::to_string(mm_rcv_connections.size()-mm_conns)
+		p.Log("Made "+std::to_string(m_args->mm_rcv_connections.size()-mm_conns)
 		   +" new middleman socket connections",3);
+		
 		// FIXME only print out new elements
 		/*
 		for(auto&& acon : clt_rtr_connections){
@@ -877,10 +950,10 @@ bool ReceiveSQL::FindNewClients_v2(){
 		
 		// update the list of clients so they can be queried
 		std::map<std::string,std::string> clientsmap;
-		for(std::pair<const std::string, Store*>& aservice : clt_rtr_connections){
+		for(std::pair<const std::string, Store*>& aservice : m_args->clt_rtr_connections){
 			clientsmap.emplace(aservice.first,"R");
 		}
-		for(std::pair<const std::string, Store*>& aservice : clt_sub_connections){
+		for(std::pair<const std::string, Store*>& aservice : m_args->clt_sub_connections){
 			if(!clientsmap.count(aservice.first)){
 				clientsmap.emplace(aservice.first,"W");
 			} else {
@@ -893,13 +966,15 @@ bool ReceiveSQL::FindNewClients_v2(){
 			if(!clientlist.empty()) clientlist+=", ";
 			clientlist += aclient.first+": "+aclient.second;
 		}
-		SC_vars["Clients"]->SetValue(clientlist);
+		p.SC_vars["Clients"]->SetValue(clientlist);
 		
 	} else {
-		Log("No new clients found",21);
+		p.Log("No new clients found",22);
 	}
 	
-	return true;
+	usleep(100000);
+	
+	return;
 }
 
 // ««-------------- ≪ °◇◆◇° ≫ --------------»»
@@ -907,14 +982,16 @@ bool ReceiveSQL::FindNewClients_v2(){
 bool ReceiveSQL::GetClientWriteQueries(){
 	
 	// see if we had any write requests from clients
-	if(in_polls.at(3).revents & ZMQ_POLLIN){
+	if(in_polls.at(2).revents & ZMQ_POLLIN){
 		Log(">>> got a write query from client",3);
 		
 		++write_queries_recvd;
 		// we did. receive next message.
 		std::vector<zmq::message_t> outputs;
 		
+		clt_sub_mtx.lock();
 		get_ok = Receive(clt_sub_socket, outputs);
+		clt_sub_mtx.unlock();
 		
 		if(not get_ok){
 			Log(Concat("error receiving part ",outputs.size()+1," of Write query from client"),1);
@@ -1021,7 +1098,7 @@ bool ReceiveSQL::GetClientWriteQueries(){
 			}
 			
 			// construct a Query object to encapsulate the query and enqueue it.
-			Query qry{outputs.at(1-o),outputs.at(2-o), database, query};
+			Query qry{outputs.at(1-o),outputs.at(2-o), database, topic, query};
 			
 			Log(Concat("QUERY WAS: '",qry.query,"'"),4);
 			wrt_txn_queue.emplace(key, qry);
@@ -1061,11 +1138,15 @@ bool ReceiveSQL::WriteDeviceConfigToQuery(const std::string& message, BStore& co
 		return false;
 	}
 	
+	// TODO Trim whitespace on all fields; in particular device name is used
+	// for versioning, so trailing whitespace will make 'LED ' different from 'LED' for e.g.
+	TrimWS(device);
+	
 	// SQL sanitization
-	get_ok  = a_database.pqxx_quote(device, device);
-	get_ok &= a_database.pqxx_quote(author, author);
-	get_ok &= a_database.pqxx_quote(description, description);
-	get_ok &= a_database.pqxx_quote(data, data);
+	get_ok  = a_database.pqxx_quote(nullptr, device, device);
+	get_ok &= a_database.pqxx_quote(nullptr, author, author);
+	get_ok &= a_database.pqxx_quote(nullptr, description, description);
+	get_ok &= a_database.pqxx_quote(nullptr, data, data);
 	if(!get_ok){
 		Log("WriteDeviceConfigToQuery: error quoting fields in message '"+message+"'",v_error);
 		return false;
@@ -1114,11 +1195,13 @@ bool ReceiveSQL::WriteRunConfigToQuery(const std::string& message, BStore& confi
 		return false;
 	}
 	
+	TrimWS(name);
+	
 	// SQL sanitization
-	get_ok  = a_database.pqxx_quote(name, name);
-	get_ok &= a_database.pqxx_quote(author, author);
-	get_ok &= a_database.pqxx_quote(description, description);
-	get_ok &= a_database.pqxx_quote(data, data);
+	get_ok  = a_database.pqxx_quote(nullptr, name, name);
+	get_ok &= a_database.pqxx_quote(nullptr, author, author);
+	get_ok &= a_database.pqxx_quote(nullptr, description, description);
+	get_ok &= a_database.pqxx_quote(nullptr, data, data);
 	if(!get_ok){
 		Log("WriteRunConfigToQuery: error quoting fields in message '"+message+"'",v_error);
 		return false;
@@ -1165,10 +1248,12 @@ bool ReceiveSQL::WriteCalibrationToQuery(const std::string& message, BStore& cal
 		return false;
 	}
 	
+	TrimWS(device);
+	
 	// SQL sanitization
-	get_ok  = a_database.pqxx_quote(device, device);
-	get_ok &= a_database.pqxx_quote(description, description);
-	get_ok &= a_database.pqxx_quote(data, data);
+	get_ok  = a_database.pqxx_quote(nullptr, device, device);
+	get_ok &= a_database.pqxx_quote(nullptr, description, description);
+	get_ok &= a_database.pqxx_quote(nullptr, data, data);
 	if(!get_ok){
 		Log("WriteCalibrationToQuery: error quoting fields in message '"+message+"'",v_error);
 		return false;
@@ -1215,9 +1300,11 @@ bool ReceiveSQL::WriteAlarmToQuery(const std::string& message, BStore& alarm, st
 		return false;
 	}
 	
+	TrimWS(device);
+	
 	// SQL sanitization
-	get_ok  = a_database.pqxx_quote(device, device);
-	get_ok &= a_database.pqxx_quote(msg, msg);
+	get_ok  = a_database.pqxx_quote(nullptr, device, device);
+	get_ok &= a_database.pqxx_quote(nullptr, msg, msg);
 	if(!get_ok){
 		Log("WriteAlarmToQuery: error quoting fields in message '"+message+"'",v_error);
 		return false;
@@ -1264,9 +1351,9 @@ bool ReceiveSQL::WriteRootPlotToQuery(const std::string& message, BStore& plot,
 	}
 	
 	// SQL sanitization
-	get_ok  = a_database.pqxx_quote(plot_name, plot_name);
-	get_ok &= a_database.pqxx_quote(draw_options, draw_options);
-	get_ok &= a_database.pqxx_quote(json_data, json_data);
+	get_ok  = a_database.pqxx_quote(nullptr, plot_name, plot_name);
+	get_ok &= a_database.pqxx_quote(nullptr, draw_options, draw_options);
+	get_ok &= a_database.pqxx_quote(nullptr, json_data, json_data);
 	if(!get_ok){
 		Log("WriteRootPlotToQuery: error quoting fields in message '"+message+"'",v_error);
 		return false;
@@ -1298,11 +1385,10 @@ bool ReceiveSQL::WriteRootPlotToQuery(const std::string& message, BStore& plot,
 bool ReceiveSQL::WritePlotToQuery(const std::string& message, BStore& plot, std::string& db_out, std::string& sql_out){
 	db_out = "daq"; // FIXME db
 	Postgres& a_database = m_databases.at(db_out);
-
+	
 	std::string name, x, y, title, xlabel, ylabel, info;
-
 	get_ok = 1;
-
+	
 	// initialize a variable with the plot slot
 #define get(slot) if (!plot.Get(#slot, slot)) goto fail_get
 	get(name);
@@ -1313,16 +1399,16 @@ bool ReceiveSQL::WritePlotToQuery(const std::string& message, BStore& plot, std:
 	get(ylabel);
 	get(info);
 #undef get
-
-        // SQL sanitization
-#define quote(slot) if (!a_database.pqxx_quote(slot, slot)) goto fail_quote
-        quote(name);
-        quote(title);
-        quote(xlabel);
-        quote(ylabel);
-        quote(info);
-#undef quote
-
+	
+	// SQL sanitization
+	#define quote(slot) if (!a_database.pqxx_quote(nullptr, slot, slot)) goto fail_quote
+	quote(name);
+	quote(title);
+	quote(xlabel);
+	quote(ylabel);
+	quote(info);
+	#undef quote
+	
 	sql_out = "INSERT INTO plots (plot, x, y, title, xlabel, ylabel, info) VALUES ("
 		+ name   + ", "
 		+ x      + ", "
@@ -1388,14 +1474,16 @@ bool ReceiveSQL::WriteMessageToQuery(const std::string& topic, const std::string
 bool ReceiveSQL::GetClientReadQueries(){
 	
 	// check if we had any read transactions dealt to us
-	if(in_polls.at(1).revents & ZMQ_POLLIN){
+	if(in_polls.at(0).revents & ZMQ_POLLIN){
 		Log(">>> got a read query from client",3);
 		
 		++read_queries_recvd;
 		// We do. receive the next query
 		std::vector<zmq::message_t> outputs;
 		
+		clt_rtr_mtx.lock();
 		get_ok = Receive(clt_rtr_socket, outputs);
+		clt_rtr_mtx.unlock();
 		
 		if(not get_ok){
 			Log(Concat("error receiving part ",outputs.size()+1," of Read query from client"),1);
@@ -1459,7 +1547,7 @@ bool ReceiveSQL::GetClientReadQueries(){
 			}
 			
 			// construct a Query object to encapsulate the query and enqueue it.
-			Query qry{outputs.at(0),outputs.at(2), database, query};
+			Query qry{outputs.at(0),outputs.at(2), database, topic, query};
 			rd_txn_queue.emplace(key, qry);
 			
 		} // else we've already got this message queued, ignore it.
@@ -1504,7 +1592,7 @@ bool ReceiveSQL::ReadDeviceConfigToQuery(const std::string& message, BStore& req
 	}
 	
 	// SQL sanitization
-	get_ok  = a_database.pqxx_quote(device, device);
+	get_ok  = a_database.pqxx_quote(nullptr, device, device);
 	if(!get_ok){
 		Log("ReadDeviceConfigToQuery: error quoting fields in message '"+message+"'",v_error);
 		return false;
@@ -1524,6 +1612,11 @@ bool ReceiveSQL::ReadDeviceConfigToQuery(const std::string& message, BStore& req
 	
 	Log(Concat("Resulting SQL: '",sql_out,"', database: '",db_out,"'"),12);
 	
+	unsigned long timestamp = time(nullptr);
+	std::string timestring;
+	TimeStringFromUnixSec(timestamp, timestring);
+	Log(Concat("Received config request '",sql_out,"' received at ",timestring),0);
+	
 	return true;
 }
 
@@ -1564,7 +1657,7 @@ bool ReceiveSQL::ReadRunConfigToQuery(const std::string& message, BStore& reques
 		}
 		
 		// SQL sanitization
-		get_ok  = a_database.pqxx_quote(config_name, config_name);
+		get_ok  = a_database.pqxx_quote(nullptr, config_name, config_name);
 		if(!get_ok){
 			Log("ReadRunConfigToQuery: error quoting fields in message '"+message+"'",v_error);
 			return false;
@@ -1605,7 +1698,7 @@ bool ReceiveSQL::ReadCalibrationToQuery(const std::string& message, BStore& requ
 	}
 	
 	// SQL sanitization
-	get_ok  = a_database.pqxx_quote(device, device);
+	get_ok  = a_database.pqxx_quote(nullptr, device, device);
 	if(!get_ok){
 		Log("ReadCalibrationToQuery: error quoting fields in message '"+message+"'",v_error);
 		return false;
@@ -1646,7 +1739,7 @@ bool ReceiveSQL::ReadRootPlotToQuery(const std::string& message, BStore& request
 	}
 	
 	// SQL sanitization
-	get_ok  = a_database.pqxx_quote(plot_name, plot_name);
+	get_ok  = a_database.pqxx_quote(nullptr, plot_name, plot_name);
 	if(!get_ok){
 		Log("ReadRootPlotToQuery: error quoting fields in message '"+message+"'",v_error);
 		return false;
@@ -1681,7 +1774,7 @@ bool ReceiveSQL::ReadPlotToQuery(const std::string& message, BStore& request, st
 	}
 
 	// SQL sanitization
-	get_ok = a_database.pqxx_quote(name, name);
+	get_ok = a_database.pqxx_quote(nullptr, name, name);
 	if (!get_ok) {
 		Log("ReadPlotToQuery: error quoting fields in message '" + message + "'", v_error);
 		return false;
@@ -1730,73 +1823,71 @@ bool ReceiveSQL::ReadMessageToQuery(const std::string& topic, const std::string&
 // ««-------------- ≪ °◇◆◇° ≫ --------------»»
 
 
-bool ReceiveSQL::GetMulticastMessages(){
+void ReceiveSQL::GetMulticastMessages(Thread_args* arg){
 	
 	// check for incoming message
+	MulticastReceive_args* m_args=reinterpret_cast<MulticastReceive_args*>(arg);
+	ReceiveSQL& p = *m_args->parent;
+	
+	if(m_args->finished) return;
+	
+	// check if parent has told us to finalise
+	if(!m_args->running){
+		// parent has told us to finalise
+		// close the socket
+		close(p.multicast_socket);
+		m_args->finished=true;
+	}
+	
+	// poll multicast socket
+	zmq::poll(&m_args->poll, 1, 100);
 	
 	// see if we had any multicast messages
-	if(in_polls.at(0).revents & ZMQ_POLLIN){
-		Log(">>> got a multicast message from client",10);
-		++multicasts_recvd;
+	if(m_args->poll.revents & ZMQ_POLLIN){
+		//p.Log(">>> got a multicast message from client",10);
+		++p.multicasts_recvd;
 		
 		// read the messge
-		char message[655355]; // theoretical maximum UDP buffer size
-		int cnt = recvfrom(multicast_socket, message, sizeof(message), 0, (struct sockaddr*)&multicast_addr, &multicast_addrlen);
-		if(cnt <= 0){
-			Log(std::string{"Failed to receive on multicast socket with error '"}+strerror(errno)+"'",v_error);
-			++multicast_recv_fails;
-			return false;
+		m_args->get_ok = recvfrom(m_args->multicast_socket, m_args->message, sizeof(m_args->message), 0, (struct sockaddr*)m_args->multicast_addr, &m_args->multicast_addrlen);
+		if(m_args->get_ok <= 0){
+			p.Log(std::string{"Failed to receive on multicast socket with error '"}+strerror(errno)+"'",p.v_error);
+			++p.multicast_recv_fails;
+			return;
 		}
 		
-		Log("Received multicast message from "+std::string{inet_ntoa(multicast_addr.sin_addr)}
-		   +": '"+std::string{message}+"'",12);
+		p.Log("Received multicast message from "+std::string{inet_ntoa(m_args->multicast_addr->sin_addr)}
+		     +": '"+std::string{m_args->message}+"'",12);
 		
-		std::string database;
-		std::string query;
-		std::string topic;
-		get_ok = MulticastMessageToQuery(message, topic, database, query);
-		
-		if(!get_ok){
-			++multicast_recv_fails;
-			return false;
-		}
-		
-		// FIXME for now all messages go to daq database,
-		// probably need to make this a pair at least with first element a DB connection or name
-		if(topic=="logging" || topic=="monitoring" || topic=="rootplot"){
-			in_multicast_queue.emplace_back(query);
-			Log("Put "+topic+" msg in queue: '"+query+"'",12);
-			
-		} else {
-			// could not determine multicast type
-			Log(std::string{"Unrecognised topic '"}+topic+"' in multicast message '"+message+"'",v_error);
-			++multicast_recv_fails;
-			return false;
-			
-		}
+		p.in_multicast_queue_mtx.lock();
+		p.in_multicast_queue.emplace_back(m_args->message);
+		p.in_multicast_queue_mtx.unlock();
+		p.Log("Put multicast msg in queue: '"+std::string(m_args->message)+"'",12);
 		
 	} /*else { std::cout<<"no multicast messages"<<std::endl; }*/
 	
-	return true;
+	p.Log("Trimming In Multicast Deque",20);
+	m_args->get_ok = p.TrimVector("in_multicast_queue");
+	
+	return;
 }
 
 // ««-------------- ≪ °◇◆◇° ≫ --------------»»
 
 // FIXME refactor to make code DRY
-bool ReceiveSQL::MulticastMessageToQuery(const std::string& message, std::string& topic_out, std::string& db_out, std::string& sql_out){
+bool ReceiveSQL::MulticastMessageToQuery(pqxx::connection* conn, const std::string& message, std::string& topic_out, std::string& db_out, std::string& sql_out, BStore& tmp, JSONP& jsonp, int& ok){
 	Log(Concat("Forming SQL for logging message: '",message,"'"),12);
 	
 	// write queries received on the pub port are JSON messages that we need to convert to SQL.
-	BStore tmp;
-	get_ok = parser.Parse(message, tmp);
-	if(!get_ok){
+	tmp.Delete();
+	ok = jsonp.Parse(message, tmp);
+	if(!ok){
 		Log("MulticastMessageToQuery error parsing message json '"+message+"'",v_error);
 		++multicast_recv_fails;
 		return false;
 	}
 	
-	get_ok = tmp.Get("topic",topic_out);
-	if(!get_ok){
+	ok = tmp.Get("topic",topic_out);
+	if(!ok){
 		++multicast_recv_fails;
 		Log("MulticastMessageToQuery error, no topic in message '"+message+"'",v_error);
 		return false;
@@ -1815,20 +1906,20 @@ bool ReceiveSQL::MulticastMessageToQuery(const std::string& message, std::string
 		uint64_t timestamp{0};
 		uint32_t severity;
 		std::string msg;
-		get_ok = tmp.Get("time",timestamp); // optional
-		get_ok = tmp.Get("device",device);
-		get_ok &= tmp.Get("severity",severity);
-		get_ok &= tmp.Get("message",msg);
-		if(!get_ok){
+		ok = tmp.Get("time",timestamp); // optional
+		ok = tmp.Get("device",device);
+		ok &= tmp.Get("severity",severity);
+		ok &= tmp.Get("message",msg);
+		if(!ok){
 			Log("MulticastMessageToQuery: missing fields in message '"+message+"'",v_error);
 			++multicast_recv_fails;
 			return false;
 		}
 		
 		// SQL sanitization
-		get_ok  = a_database.pqxx_quote(device, device);
-		get_ok &= a_database.pqxx_quote(msg, msg);
-		if(!get_ok){
+		ok  = a_database.pqxx_quote(conn, device, device);
+		ok &= a_database.pqxx_quote(conn, msg, msg);
+		if(!ok){
 			Log("MulticastMessageToQuery: error quoting fields in message '"+message+"'",v_error);
 			++multicast_recv_fails;
 			return false;
@@ -1838,15 +1929,16 @@ bool ReceiveSQL::MulticastMessageToQuery(const std::string& message, std::string
 		// build an ISO 8601 timestamp ("2015-10-02 11:16:34+0100")
 		// (the trailing "+0100" is number of [hours][mins] in local timezone relative to UTC)
 		std::string timestring;
-		get_ok = TimeStringFromUnixMs(timestamp, timestring);
-		if(!get_ok) timestring="now()"; // since multicast doesn't propagate back an error, assume now
+		ok = TimeStringFromUnixMs(timestamp, timestring);
+		if(!ok) timestring="now()"; // since multicast doesn't propagate back an error, assume now
 		
 		// form into a suitable SQL query
-		sql_out = "INSERT INTO logging ( time, device, severity, message ) VALUES ( '"
+		//sql_out = "INSERT INTO logging ( time, device, severity, message ) VALUES ( '"
+		sql_out = "( '"                                             // XXX PARTIAL SQL FOR BATCH INSERTION!!!
 		        + timestring               + "',"
 		        + device                   + ","
 		        + std::to_string(severity) + ","
-		        + msg                      + ");";
+		        + msg                      + ")";
 		
 		Log(Concat("Resulting SQL: '",sql_out,"', database: '",db_out,"', topic: ",topic_out),100);
 		
@@ -1863,32 +1955,33 @@ bool ReceiveSQL::MulticastMessageToQuery(const std::string& message, std::string
 		uint64_t timestamp{0};
 		std::string data;
 		tmp.Get("time",timestamp); // optional
-		get_ok = tmp.Get("device",device);
-		get_ok &= tmp.Get("data",data);
-		if(!get_ok){
+		ok = tmp.Get("device",device);
+		ok &= tmp.Get("data",data);
+		if(!ok){
 			Log("MulticastMessageToQuery: missing fields in message '"+message+"'",v_error);
 			++multicast_recv_fails;
 			return false;
 		}
 		
 		// SQL sanitization
-		get_ok  = a_database.pqxx_quote(device, device);
-		get_ok &= a_database.pqxx_quote(data, data);
-		if(!get_ok){
+		ok  = a_database.pqxx_quote(conn, device, device);
+		ok &= a_database.pqxx_quote(conn, data, data);
+		if(!ok){
 			Log("MulticastMessageToQuery: error quoting fields in message '"+message+"'",v_error);
 			++multicast_recv_fails;
 			return false;
 		}
 		
 		std::string timestring;
-		get_ok = TimeStringFromUnixMs(timestamp, timestring);
-		if(!get_ok) timestring="now()"; // since multicast doesn't propagate back an error, assume now
+		ok = TimeStringFromUnixMs(timestamp, timestring);
+		if(!ok) timestring="now()"; // since multicast doesn't propagate back an error, assume now
 		
 		// form into a suitable SQL query
-		sql_out = "INSERT INTO monitoring ( time, device, data ) VALUES ( '"
+		//sql_out = "INSERT INTO monitoring ( time, device, data ) VALUES ( '"
+		sql_out = "( '"                                      // XXX PARTIAL SQL FOR BATCH INSERTION!!!
 		        + timestring + "',"
 		        + device     + ","
-		        + data       + ");";
+		        + data       + ")";
 		
 		Log(Concat("Resulting SQL: '",sql_out,"', database: '",db_out,"', topic: ",topic_out),50);
 		
@@ -1906,28 +1999,28 @@ bool ReceiveSQL::MulticastMessageToQuery(const std::string& message, std::string
 		std::string data;
 		std::string draw_options;
 		tmp.Get("time",timestamp); // optional
-		get_ok = tmp.Get("plot_name",plot_name);
-		get_ok &= tmp.Get("data",data);
-		get_ok &= tmp.Get("draw_options",draw_options);
-		if(!get_ok){
+		ok = tmp.Get("plot_name",plot_name);
+		ok &= tmp.Get("data",data);
+		ok &= tmp.Get("draw_options",draw_options);
+		if(!ok){
 			Log("MulticastMessageToQuery: missing fields in message '"+message+"'",v_error);
 			++multicast_recv_fails;
 			return false;
 		}
 		
 		// SQL sanitization
-		get_ok  = a_database.pqxx_quote(plot_name, plot_name);
-		get_ok &= a_database.pqxx_quote(draw_options, draw_options);
-		get_ok &= a_database.pqxx_quote(data, data);
-		if(!get_ok){
+		ok  = a_database.pqxx_quote(conn, plot_name, plot_name);
+		ok &= a_database.pqxx_quote(conn, draw_options, draw_options);
+		ok &= a_database.pqxx_quote(conn, data, data);
+		if(!ok){
 			Log("MulticastMessageToQuery: error quoting fields in message '"+message+"'",v_error);
 			++multicast_recv_fails;
 			return false;
 		}
 		
 		std::string timestring;
-		get_ok = TimeStringFromUnixMs(timestamp, timestring);
-		if(!get_ok) timestring="now()"; // since multicast doesn't propagate back an error, assume now
+		ok = TimeStringFromUnixMs(timestamp, timestring);
+		if(!ok) timestring="now()"; // since multicast doesn't propagate back an error, assume now
 		
 		// form into a suitable SQL query
 		// FIXME this needs to insert into a temporary root plots table
@@ -1967,11 +2060,13 @@ bool ReceiveSQL::GetMiddlemanCheckin(){
 	// keep reading from the SUB socket until there are no more waiting messages.
 	// it's important we don't take any action until we've read out everything,
 	// to ensure we don't start negotiation based on old, stale requests.
-	while(in_polls.at(2).revents & ZMQ_POLLIN){
+	while(in_polls.at(1).revents & ZMQ_POLLIN){
 		
 		// We do. Receive it.
 		std::vector<zmq::message_t> outputs;
+		mm_rcv_mtx.lock();
 		get_ok = Receive(mm_rcv_socket, outputs);
+		mm_rcv_mtx.unlock();
 		if(not get_ok){
 			Log(Concat("error receiving message part ",outputs.size()+1," of message from middleman"),0);
 			++mm_broadcast_recv_fails;  // FIXME this includes negotiation requests and our own broadcasts
@@ -2059,7 +2154,7 @@ bool ReceiveSQL::GetMiddlemanCheckin(){
 		// but perhaps this message is stale:
 		// re-poll the socket and see if there is another message in the buffer
 		try {
-			get_ok = zmq::poll(&in_polls.at(2), 1, 0);
+			get_ok = zmq::poll(&in_polls.at(1), 1, 0);
 		} catch (zmq::error_t& err){
 			std::cerr<<"ReceiveSQL::GetMiddlemanCheckin poller caught "<<err.what()<<std::endl;
 			get_ok = -1;
@@ -2140,9 +2235,9 @@ bool ReceiveSQL::RunNextWriteQuery(){
 			return false;
 		}
 		std::string err;
-		next_msg.query_ok = m_databases.at(db).QueryAsJsons(next_msg.query, &next_msg.response, &err);
+		next_msg.query_ok = m_databases.at(db).QueryAsJsons(nullptr, next_msg.query, &next_msg.response, &err);
 		if(not next_msg.query_ok){
-			Log(Concat("Write query failed! Query was: \"",next_msg.query,"\", error was: '",err,"'"),1);
+			Log(Concat("Write query failed! Query was: \"",next_msg.query,"\", error was: '",err,"'"),0);
 			++write_queries_failed;
 			next_msg.response = std::vector<std::string>{err};
 		}
@@ -2172,9 +2267,18 @@ bool ReceiveSQL::RunNextReadQuery(){
 			return false;
 		}
 		std::string err;
-		next_msg.query_ok = m_databases.at(db).QueryAsJsons(next_msg.query, &next_msg.response, &err);
+		next_msg.query_ok = m_databases.at(db).QueryAsJsons(nullptr, next_msg.query, &next_msg.response, &err);
+		
+		// XXX debug
+		if(next_msg.topic=="DEVCONFIG"){  // FINDME
+			unsigned long timestamp = time(nullptr);
+			std::string timestring;
+			TimeStringFromUnixSec(timestamp, timestring);
+			Log(Concat("Ran config request query '",next_msg.query,"' at ",timestring," result: ",next_msg.query_ok),0);
+		}
+		
 		if(not next_msg.query_ok){
-			Log(Concat("Read query failed! Query was: \"",next_msg.query,"\", error was: '",err,"'"),1);
+			Log(Concat("Read query failed! Query was: \"",next_msg.query,"\", error was: '",err,"'"),0);
 			++read_queries_failed;
 			next_msg.response = std::vector<std::string>{err};
 		}
@@ -2191,37 +2295,111 @@ bool ReceiveSQL::RunNextReadQuery(){
 
 // ««-------------- ≪ °◇◆◇° ≫ --------------»»
 
-bool ReceiveSQL::RunNextMulticastMsg(){
+void ReceiveSQL::MulticastWorker(Thread_args* arg){
+	
+	MulticastWorker_args* m_args=reinterpret_cast<MulticastWorker_args*>(arg);
+	ReceiveSQL& p = *m_args->parent;
+	
+	if(m_args->finished) return;
 	
-	// insert our next fire-and-forget message, if we have one
-	if(in_multicast_queue.size()){
-		Log("Inserting next multicast message to DB: we have "+std::to_string(in_multicast_queue.size())
-		    +" messages to process",13);
+	// check if parent has new messages for us
+	std::vector<std::string> multicast_msg_queue;
+	p.in_multicast_queue_mtx.lock();
+	if(p.in_multicast_queue.empty()){
+		p.in_multicast_queue_mtx.unlock();
+		
+		// check if parent has told us to finalise
+		if(!m_args->running){
+			m_args->finished=true;
+		}
 		
-		std::string next_msg = in_multicast_queue.front();
-		auto t_start = std::chrono::high_resolution_clock::now();
-		get_ok = m_databases.at("daq").Query(next_msg);  // FIXME hard-coded db name
+		usleep(100);  // don't keep locking it too often
+		return;
+		
+	}
+	
+	// parent has messages for us. grab the message queue, swapping to give them a new one
+	std::swap(p.in_multicast_queue, multicast_msg_queue);
+	p.in_multicast_queue_mtx.unlock();
+	
+	p.multicasts_waiting = multicast_msg_queue.size();
+	p.Log("Inserting next multicast message to DB: we have "+std::to_string(multicast_msg_queue.size())
+	     +" messages to process",13);
+	
+	// loop over and run each query
+	for(std::string& next_msg : multicast_msg_queue){
+		
+		m_args->get_ok = p.MulticastMessageToQuery(m_args->conn, next_msg, m_args->topic, m_args->database, m_args->query, m_args->tmp, m_args->jsonp, m_args->get_ok);
+		
+		if(!m_args->get_ok){
+			++p.multicast_recv_fails;
+			continue;
+		}
+		
+		// FIXME for now all messages go to daq database,
+		// probably need to make this a pair at least with first element a DB connection or name
+		if(m_args->topic!="logging" && m_args->topic!="monitoring" && m_args->topic!="rootplot"){
+			// could not determine multicast type
+			p.Log(std::string{"Unrecognised topic '"}+m_args->topic+"' in multicast message '"+next_msg+"'",p.v_error);
+			++p.multicast_recv_fails;
+			continue;
+		} else if(m_args->topic=="logging"){
+			m_args->logmsgs.push_back(m_args->query);
+		} else {
+			m_args->monmsgs.push_back(m_args->query);
+		}
+		
+	}
+	
+	// merge all messages and do batch insertion
+	std::vector<std::string>* msgset=nullptr;
+	boost::posix_time::ptime* last_insert=nullptr;
+	for(int i=0; i<2; ++i){
+		if(i==0){
+			msgset = &m_args->logmsgs;
+			m_args->query = "INSERT INTO logging ( time, device, severity, message ) VALUES ";
+			last_insert = &m_args->last_log_insert;
+		} else {
+			msgset = &m_args->monmsgs;
+			m_args->query = "INSERT INTO monitoring ( time, device, data ) VALUES ";
+			last_insert = &m_args->last_mon_insert;
+		}
+		size_t nmsgs = msgset->size();
+		boost::posix_time::time_duration lapse = *last_insert - boost::posix_time::microsec_clock::universal_time();
+		if(nmsgs < p.batch_size && lapse.total_seconds() < 30) continue;
+		
+		for(size_t i=0; i<nmsgs; ++i){
+			if(i!=0) m_args->query += ", ";
+			m_args->query += msgset->at(i);
+		}
+		m_args->query += ";";
+		msgset->resize(0);
+		
+		// run a single batch insertion to minimise number of transactions
+		//auto t_start = std::chrono::high_resolution_clock::now();
+		m_args->get_ok = p.m_databases.at(m_args->database).Query(m_args->conn, m_args->query);  // FIXME hard-coded db name
+		p.Log("batch insert of "+std::to_string(nmsgs)+" multicasts",2);
+		*last_insert = boost::posix_time::microsec_clock::universal_time();
+		/*
 		auto t_end = std::chrono::high_resolution_clock::now();
 		if(std::chrono::duration_cast<std::chrono::milliseconds>(t_end-t_start).count()>1000){
-			Log("Slow multicast query: '"+next_msg+"' took "+
-			std::to_string(std::chrono::duration_cast<std::chrono::milliseconds>(t_end-t_start).count())+" ms",v_warning);
+			//p.Log("Slow multicast query: '"+m_args->query+"' took "+
+			p.Log("Slow multicast query took "+
+			std::to_string(std::chrono::duration_cast<std::chrono::milliseconds>(t_end-t_start).count())+" ms",p.v_warning);
 		}
+		*/
 		
-		if(not get_ok){
+		if(not m_args->get_ok){
 			// something went wrong
-			std::cerr<<"Error inserting logmessage '"<<next_msg<<"' into database"<<std::endl;
-			// can't use Log or we end up in a circular loop
-			in_multicast_queue.pop_front();
-			++multicast_queries_failed;
-			return false;
+			std::cerr<<"Error inserting logmessage '"<<m_args->query<<"' into database"<<std::endl;
+			// can't use Log or we end up in a circular loop since Log is a multicast query itself.
+			p.multicast_queries_failed+=nmsgs;   // XXX FIXME these trackers won't work any more with batch insertion
 		}
-		
-		// remove the message from the queue
-		in_multicast_queue.pop_front();
-		
-	} // else no log messages for now
+		p.multicasts_waiting-=nmsgs;
+	}
 	
-	return true;
+	
+	return;
 }
 
 // ««-------------- ≪ °◇◆◇° ≫ --------------»»
@@ -2234,7 +2412,7 @@ bool ReceiveSQL::SendNextReply(){
 	if(resp_queue.size()){
 		
 		// check we had a listener ready
-		if(out_polls.at(1).revents & ZMQ_POLLOUT){
+		if(out_polls.at(0).revents & ZMQ_POLLOUT){
 			
 			// OK to send! Get the message to acknowledge
 			Query& next_msg = resp_queue.begin()->second;
@@ -2246,13 +2424,14 @@ bool ReceiveSQL::SendNextReply(){
 			
 			std::string client_str(reinterpret_cast<char*>(next_msg.client_id.data()));
 			uint32_t* msgID = reinterpret_cast<uint32_t*>(next_msg.message_id.data());
-			Log("Sending next reply to ZMQ IDENTITY '"+client_str+"' for msg "+std::to_string(*msgID),1);
+			Log("Sending next reply to ZMQ IDENTITY '"+client_str+"' for msg "+std::to_string(*msgID),12);
 			
 			// as soon as we send a zmq::message_t (i.e. client_id and message_id), they are "used up":
 			// the 'message.size()' becomes 0 and they strictly no longer retain their contents.
 			// to keep a copy cached for re-sending we need to explicitly make a copy now.
 			Query qrycpy(next_msg);  // (the Query copy-constructor invokes zmq::message_t->copy on members)
 			
+			clt_rtr_mtx.lock();
 			if(next_msg.response.size()==0){
 				try{
 					get_ok = Send(clt_rtr_socket,
@@ -2277,6 +2456,7 @@ bool ReceiveSQL::SendNextReply(){
 					get_ok=false;
 				}
 			}
+			clt_rtr_mtx.unlock();
 			
 			if(get_ok){
 				// all parts sent successfully, add to the sent cache
@@ -2297,6 +2477,13 @@ bool ReceiveSQL::SendNextReply(){
 				}
 			} // end send ok check
 			
+			if(qrycpy.topic=="DEVCONFIG"){
+				unsigned long timestamp = time(nullptr);
+				std::string timestring;
+				TimeStringFromUnixSec(timestamp, timestring);
+				Log(Concat("Replied config request '",qrycpy.query,"' at ",timestring," result: ",get_ok),0);
+			}
+			
 		} // else no available listeners
 		
 	} // else no responses to send
@@ -2316,7 +2503,7 @@ bool ReceiveSQL::SendNextMulticast(){
 	if(out_multicast_queue.size()){
 		
 		// check we had a listener ready
-		if(out_polls.at(0).revents & ZMQ_POLLOUT){
+		if(true){  // not relevnt for sending multicast?
 			
 			// OK to send! Get the message
 			std::string& message = out_multicast_queue.front();
@@ -2352,7 +2539,7 @@ bool ReceiveSQL::BroadcastPresence(){
 	
 	if(elapsed_time.is_negative()){
 		
-		if(out_polls.at(2).revents & ZMQ_POLLOUT){
+		if(out_polls.at(1).revents & ZMQ_POLLOUT){
 			
 			++mm_broadcasts_sent;
 			uint32_t msg = am_master;
@@ -2444,15 +2631,12 @@ bool ReceiveSQL::TrimQueue(const std::string& queuename){
 
 bool ReceiveSQL::TrimDequeue(const std::string& queuename){
 	
-	// check in or out log message queue size and do the same
+	// check message dequeue size and do the same
 	std::deque<std::string>* queue;
 	unsigned long* drop_count;
 	
 	// check which queue we're managing
-	if(queuename=="in_multicast_queue"){
-		queue = &in_multicast_queue;
-		drop_count = &dropped_multicast_in;
-	} else if(queuename=="out_multicast_queue"){
+	if(queuename=="out_multicast_queue"){
 		queue = &out_multicast_queue;
 		drop_count = &dropped_logs_out;
 	} else {
@@ -2480,6 +2664,51 @@ bool ReceiveSQL::TrimDequeue(const std::string& queuename){
 
 // ««-------------- ≪ °◇◆◇° ≫ --------------»»
 
+bool ReceiveSQL::TrimVector(const std::string& queuename){
+	
+	// check message vector size and do the same
+	std::vector<std::string>* queue;
+	unsigned long* drop_count;
+	
+	// check which queue we're managing
+	if(queuename=="in_multicast_queue"){
+		queue = &in_multicast_queue;
+		drop_count = &dropped_multicast_in;
+	} else {
+		Log(Concat("TrimVector called with unknown message queue '",queuename,"'"),0);
+		return false;
+	}
+	
+	// FIXME this trimming process needs to be done better for background threads
+	// since we have some messages in the thread and some in the parent,
+	// so we need to trim in two places and use mutexes when handling parent one....
+	in_multicast_queue_mtx.lock();
+	
+	// FIXME better than queue size would be to look at messages older than a certain time
+	// and trim based on that, since how well we're keeping up is more about how long
+	// each query takes, not about how many we have waiting.
+	if(queue->size() > drop_limit){
+		int to_drop = queue->size() - drop_limit;
+		in_multicast_queue.erase(in_multicast_queue.begin(), in_multicast_queue.begin()+to_drop);
+		in_multicast_queue_mtx.unlock();
+		
+		Log(Concat("Warning! Number of waiting elements in queue ",queuename," (",queue->size(),
+		           ") is over limit (",drop_limit,")! Dropping ",to_drop," messages!"),0);
+		*drop_count += to_drop;
+		
+	// check if we need to warn about being close to the limit
+	} else if(queue->size() > warn_limit){
+		Log(Concat("Warning! Number of waiting elements in ",queuename," (",queue->size(),
+		           ") is approaching drop limit (",drop_limit,") !",
+		           "Is the network down, or responding slowly?"),1);
+	}
+	in_multicast_queue_mtx.unlock();
+	
+	return true;
+}
+
+// ««-------------- ≪ °◇◆◇° ≫ --------------»»
+
 bool ReceiveSQL::TrimCache(){
 	
 	// check cache size and do the same
@@ -2526,6 +2755,10 @@ bool ReceiveSQL::UpdateControls(){
 	SC_vars["Quit"]->GetValue(quit);
 	if(quit) DoQuit(quit);
 	
+	bool clearconnections=false;
+	SC_vars["ClearConnections"]->GetValue(clearconnections);
+	if(clearconnections) ClearConnections(clearconnections);
+	
 	return true;
 }
 
@@ -2542,8 +2775,10 @@ bool ReceiveSQL::TrackStats(){
 		// of reads/writes since last time. So get the last values
 		unsigned long last_write_query_count;
 		unsigned long last_read_query_count;
+		unsigned long last_multicast_count;
 		MonitoringStore.Get("write_queries_recvd", last_write_query_count);
 		MonitoringStore.Get("read_queries_recvd", last_read_query_count);
+		MonitoringStore.Get("multicasts_recvd", last_multicast_count);
 		
 		// calculate rates are per minute
 		elapsed_time = boost::posix_time::microsec_clock::universal_time() - last_stats_calc;
@@ -2552,6 +2787,8 @@ bool ReceiveSQL::TrackStats(){
 		    ((read_queries_recvd - last_read_query_count) * 60.) / elapsed_time.total_seconds();
 		float write_query_rate = (elapsed_time.total_seconds()==0) ? 0 :
 		    ((write_queries_recvd - last_write_query_count) * 60.) / elapsed_time.total_seconds();
+		float multicast_rate = (elapsed_time.total_seconds()==0) ? 0 :
+		    ((multicasts_recvd - last_multicast_count) * 60.) / elapsed_time.total_seconds();
 		
 		// dump all stats into a Store.
 		MonitoringStore.Set("min_loop_time",min_loop_ms);
@@ -2623,12 +2860,21 @@ bool ReceiveSQL::TrackStats(){
 		db_verbosity = db_verbosity_tmp;
 		*/
 		if(am_master){
+			/*
 			std::string sql_qry = "INSERT INTO monitoring ( time, device, data ) VALUES ( 'now()', '"
 			                    + my_id+"', '"+json_stats+"' );";
-			in_multicast_queue.push_back(sql_qry);
+			*/
+			
+			std::string multicast_msg = "{ \"topic\":\"monitoring\""
+			                            ", \"device\":\""+escape_json(my_id)+"\""
+			                          + ", \"time\":"+std::to_string(time(nullptr)*1000)  // ms since unix epoch
+			                          + ", \"data\":\""+json_stats+"\" }";
+			in_multicast_queue_mtx.lock();
+			in_multicast_queue.push_back(multicast_msg);
+			in_multicast_queue_mtx.unlock();
 		} else {
 			std::string multicast_msg = "{ \"topic\":\"monitoring\""
-			                            ", \"device\":\""+escape_json(my_id)+"\", "
+			                            ", \"device\":\""+escape_json(my_id)+"\""
 			                          + ", \"time\":"+std::to_string(time(nullptr)*1000)  // ms since unix epoch
 			                          + ", \"data\":\""+json_stats+"\" }";
 			out_multicast_queue.push_back(multicast_msg);
@@ -2707,7 +2953,7 @@ bool ReceiveSQL::NegotiationRequest(){
 		if(first_send || elapsed_time.is_negative()){
 			
 			// send out our message
-			int ret = PollAndSend(mm_snd_socket, out_polls.at(2), outpoll_timeout, my_id, our_header, our_timestamp);
+			int ret = PollAndSend(mm_snd_socket, out_polls.at(1), outpoll_timeout, my_id, our_header, our_timestamp);
 			
 			// check for errors
 			if(ret==-3) Log("Error polling out socket in NegotiateMaster() call!",0);
@@ -2725,7 +2971,9 @@ bool ReceiveSQL::NegotiationRequest(){
 		// receive the other middleman's response
 		std::vector<zmq::message_t> messages;
 		
-		int ret = PollAndReceive(mm_rcv_socket, in_polls.at(2), inpoll_timeout, messages);
+		mm_rcv_mtx.lock();
+		int ret = PollAndReceive(mm_rcv_socket, in_polls.at(1), inpoll_timeout, messages);
+		mm_rcv_mtx.unlock();
 		
 		// chech for errors
 		if(ret==-3) Log("Error polling in socket in NegotiateMaster() call!",0);
@@ -2833,7 +3081,7 @@ bool ReceiveSQL::NegotiationRequest(){
 		// they must've opened negotiations at the same time we did. They may be expecting a reply.
 		
 		// send the reply
-		int ret = PollAndSend(mm_snd_socket, out_polls.at(2), outpoll_timeout, my_id, our_header, our_timestamp);
+		int ret = PollAndSend(mm_snd_socket, out_polls.at(1), outpoll_timeout, my_id, our_header, our_timestamp);
 		
 		// handle errors
 		if(ret==-3) Log("Error polling out socket in NegotiateMaster() call!",0);
@@ -2892,7 +3140,7 @@ bool ReceiveSQL::NegotiationReply(const std::string& their_header, const std::st
 	}
 	
 	// inform the other middleman
-	int ret = PollAndSend(mm_snd_socket, out_polls.at(2), 500, my_id, our_header, our_timestamp);
+	int ret = PollAndSend(mm_snd_socket, out_polls.at(1), 500, my_id, our_header, our_timestamp);
 	
 	// handle errors
 	if(ret==-3) Log("Error polling out socket in NegotiateMaster() call!",0);
@@ -2932,7 +3180,7 @@ bool ReceiveSQL::UpdateRole(){
 		
 		// promote the database out of recovery mode. 60s timeout.
 		std::string err;
-		get_ok = m_databases.begin()->second.Promote(60,&err); // FIXME multiple dbs?
+		get_ok = m_databases.begin()->second.Promote(nullptr, 60,&err); // FIXME multiple dbs?
 		
 		// should we also stop broadcasting ourself as a source of logging messages?
 		utilities->RemoveService("logging");
@@ -2967,7 +3215,7 @@ bool ReceiveSQL::UpdateRole(){
 		std::string err;
 		
 		// demote the database to standby. 60s timeout.
-		get_ok = m_databases.begin()->second.Demote(60,&err); // FIXME multiple dbs?
+		get_ok = m_databases.begin()->second.Demote(nullptr, 60,&err); // FIXME multiple dbs?
 		
 		// check for errors
 		if(get_ok){
@@ -3077,7 +3325,7 @@ bool ReceiveSQL::GetLastUpdateTime(std::string& our_timestamp){
 	std::string err;
 	std::vector<std::string> results;
 	
-	bool query_ok = m_databases.begin()->second.QueryAsStrings(query, &results, 'r', &err); // FIXME multiple dbs?
+	bool query_ok = m_databases.begin()->second.QueryAsStrings(nullptr, query, &results, 'r', &err); // FIXME multiple dbs?
 	
 	if(not query_ok || results.size()==0){
 		Log(Concat("Error getting last commit timestamp in negotiation! ",
@@ -3093,13 +3341,13 @@ bool ReceiveSQL::GetLastUpdateTime(std::string& our_timestamp){
 
 // ««-------------- ≪ °◇◆◇° ≫ --------------»»
 
-bool ReceiveSQL::TimeStringFromUnixSec(uint64_t timestamp, std::string& timestring){
+bool ReceiveSQL::TimeStringFromUnixSec(const uint64_t& timestamp, std::string& timestring){
 	return TimeStringFromUnixMs(timestamp*1000, timestring);
 }
 
 // ««-------------- ≪ °◇◆◇° ≫ --------------»»
 
-bool ReceiveSQL::TimeStringFromUnixMs(uint64_t timestamp, std::string& timestring){
+bool ReceiveSQL::TimeStringFromUnixMs(const uint64_t& timestamp, std::string& timestring){
 	
 	if(timestamp==0){
 		timestring="now()";
@@ -3107,7 +3355,7 @@ bool ReceiveSQL::TimeStringFromUnixMs(uint64_t timestamp, std::string& timestrin
 		//std::cout<<"converting time "<<timestamp<<" to timestring"<<std::endl;
 		timestring.resize(23, '\0');
 		int timestamp_ms = timestamp%1000;
-		time_t timestamp_sec = timestamp*0.001;  // time_t is equivalent to uint64_t
+		time_t timestamp_sec = timestamp/1000;  // time_t is equivalent to uint64_t
 		struct tm* timeptr = gmtime(&timestamp_sec);
 		//std::cout<<"timeptr is "<<timeptr<<std::endl;
 		if(timeptr==0){
@@ -3163,6 +3411,21 @@ std::string ReceiveSQL::escape_json(std::string s){
 	return s;
 }
 
+inline void ReceiveSQL::TrimWS(std::string& s){
+	
+	// trim from start (in place)
+	s.erase(s.begin(), std::find_if(s.begin(), s.end(), [](unsigned char ch) {
+		return !std::isspace(ch);
+	}));
+	
+	// trim from end (in place)
+	s.erase(std::find_if(s.rbegin(), s.rend(), [](unsigned char ch) {
+		return !std::isspace(ch);
+	}).base(), s.end());
+	
+	return;
+}
+
 //                   ≫ ──── ≪•◦ ❈ ◦•≫ ──── ≪
 //                      generic ZMQ wrappers
 //                   ≫ ──── ≪•◦ ❈ ◦•≫ ──── ≪
@@ -3316,6 +3579,8 @@ bool ReceiveSQL::Log(const std::string& message, uint32_t message_severity){
 	if(message_severity < db_verbosity){
 		// we'll either want to run this locally, or send it to the master, depending on our role
 		if(am_master){
+			
+			/*  XXX JSON decoding is done by multicast workers now
 			// queue up for logging to our local monitoring database
 			if(m_databases.empty()){
 				// right now we only need the connection for sanitizing the message.
@@ -3331,7 +3596,7 @@ bool ReceiveSQL::Log(const std::string& message, uint32_t message_severity){
 			
 			// SQL sanitization
 			std::string msg;
-			get_ok = m_databases.begin()->second.pqxx_quote(message, msg);
+			get_ok = m_databases.begin()->second.pqxx_quote(nullptr, message, msg);
 			if(!get_ok){
 				std::cerr<<"Error sanitizing log message '"<<message<<"'"<<std::endl;
 				// can't use Log here as the embedded message would result in a circular loop!
@@ -3344,17 +3609,27 @@ bool ReceiveSQL::Log(const std::string& message, uint32_t message_severity){
 			                   + my_id                            + "',"
 			                   + std::to_string(message_severity) + ","
 			                   + msg                              + ");";
+			*/
 			
+			// form the required JSON
+			std::string logmsg = "{ \"topic\":\"logging\""
+			                     ", \"time\":"+std::to_string(time(nullptr)*1000)  // ms since unix epoch
+			                    +", \"device\":\""+escape_json(my_id)+"\""
+			                    +", \"severity\":"+std::to_string(message_severity)
+			                    +", \"message\":\""+escape_json(message)+"\" }";
+			
+			in_multicast_queue_mtx.lock();
 			in_multicast_queue.emplace_back(logmsg);
+			in_multicast_queue_mtx.unlock();
 			
 		} else {
 			// add to the queue of logging messages to send to the master over ZMQ
 			
 			// form the required JSON
 			std::string logmsg = "{ \"time\":"+std::to_string(time(nullptr)*1000)  // ms since unix epoch
-			                   +", \"device\":\""+escape_json(my_id)+"\""
-			                   +", \"severity\":"+std::to_string(message_severity)
-			                   +", \"message\":\""+escape_json(message)+"\" }";
+			                    +", \"device\":\""+escape_json(my_id)+"\""
+			                    +", \"severity\":"+std::to_string(message_severity)
+			                    +", \"message\":\""+escape_json(message)+"\" }";
 			
 			out_multicast_queue.emplace_back(logmsg);
 		}
@@ -3390,6 +3665,17 @@ bool ReceiveSQL::DoQuit(bool quit){
 	return true;
 }
 
+bool ReceiveSQL::ClearConnections(bool clearconnections){
+	if(!clearconnections) return true;
+	bool ok=true;
+	/* connection maps are now in FindNewClients thread args
+	ok = ok && utilities->ClearConnections(clt_rtr_socket, clt_rtr_connections);
+	ok = ok && utilities->ClearConnections(clt_sub_socket, clt_sub_connections);
+	ok = ok && utilities->ClearConnections(mm_rcv_socket, mm_rcv_connections);
+	*/
+	return ok;
+}
+
 //https://wiki.postgresql.org/wiki/What%27s_new_in_PostgreSQL_9.5#Commit_timestamp_tracking
 //https://stackoverflow.com/questions/33943524/atomically-set-serial-value-when-committing-transaction/33944402#33944402
 //https://dba.stackexchange.com/questions/199290/get-last-modified-date-of-table-in-postgresql
diff --git a/ReceiveSQL.h b/ReceiveSQL.h
index c6a62d3..678cc57 100755
--- a/ReceiveSQL.h
+++ b/ReceiveSQL.h
@@ -10,6 +10,8 @@
 // for finding clients
 #include "ServiceDiscovery.h"
 #include "MMUtilities.h"
+// for background thread
+#include "Utilities.h"
 // for slow control over zmq SD port
 #include "SlowControlCollection.h"
 // for databse interaction
@@ -34,6 +36,55 @@
 #include <arpa/inet.h>
 #include <netinet/in.h>
 #include <fcntl.h>
+// for thread safety
+#include <mutex>
+
+class ReceiveSQL;
+
+// POD class for things passed to thread to connect to new client
+struct FindClients_args : public Thread_args {
+	ReceiveSQL* parent;
+	MMUtilities* utilities;
+	// required by the Utilities class to keep track of connections to clients
+	// we should have one map per zmq_socket managed by the Utilities class;
+	// it uses this to determine if we are connected to a given client already
+	std::map<std::string,Store*> clt_rtr_connections;
+	std::map<std::string,Store*> mm_rcv_connections;
+	std::map<std::string,Store*> clt_sub_connections;
+};
+
+// POD class for things passed to multicast worker threads
+struct MulticastWorker_args : public Thread_args {
+	ReceiveSQL* parent;
+	bool running;
+	bool finished;
+	// thread must maintain its own connection to database
+	pqxx::connection* conn;
+	std::mutex busy;
+	BStore tmp;
+	JSONP jsonp;
+	std::string database;
+	std::string query;
+	std::string topic;
+	int get_ok;
+	std::vector<std::string> logmsgs;
+	std::vector<std::string> monmsgs;
+	boost::posix_time::ptime last_log_insert;
+	boost::posix_time::ptime last_mon_insert;
+};
+
+// POD class for things passed to multicast listener thread
+struct MulticastReceive_args : public Thread_args {
+	ReceiveSQL* parent;
+	bool running;
+	bool finished;
+	socklen_t multicast_addrlen;
+	struct sockaddr_in* multicast_addr;
+	int multicast_socket;
+	zmq::pollitem_t poll;
+	char message[655355]; // theoretical maximum UDP buffer size
+	int get_ok;
+};
 
 class ReceiveSQL{
 	public:
@@ -50,29 +101,38 @@ class ReceiveSQL{
 	
 	bool Execute();
 	bool FindNewClients();
-	bool FindNewClients_v2();
+	static void FindNewClients_v2(Thread_args* args);
 	bool GetClientWriteQueries();
 	bool WriteMessageToQuery(const std::string& topic, const std::string& message, std::string& db_out, std::string& sql_out);
 	bool GetClientReadQueries();
 	bool ReadMessageToQuery(const std::string& topic, const std::string& message, std::string& db_out, std::string& sql_out);
-	bool GetMulticastMessages();
-	bool MulticastMessageToQuery(const std::string& message, std::string& topic_out, std::string& db_out, std::string& sql_out);
+	//bool GetMulticastMessages();
+	static void GetMulticastMessages(Thread_args* args); // run multicast queries
+	bool MulticastMessageToQuery(pqxx::connection* conn, const std::string& message, std::string& topic_out, std::string& db_out, std::string& sql_out, BStore& tmp, JSONP& jsonp, int& ok);
 	bool GetMiddlemanCheckin();
 	bool CheckMasterStatus();
 	bool RunNextWriteQuery();
 	bool RunNextReadQuery();
-	bool RunNextMulticastMsg();
+	//bool RunNextMulticastMsg();
+	static void MulticastWorker(Thread_args* args); // receive multicast messages
+	std::vector<MulticastWorker_args*> multicast_worker_args;
+	MulticastReceive_args* multicast_listener_args=nullptr;
+	FindClients_args* findclients_args=nullptr;
+	size_t batch_size = 100; // minimum number of multicast messages to buffer before insert (or >30s)
 	bool SendNextReply();
 	bool SendNextMulticast();
 	std::string escape_json(std::string s);
+	void TrimWS(std::string&);
 	bool BroadcastPresence();
 	bool CleanupCache();
 	bool TrimQueue(const std::string& queuename);
 	bool TrimDequeue(const std::string& queuename);
+	bool TrimVector(const std::string& queuename);
 	bool TrimCache();
 	bool UpdateControls();
 	bool DoStop(bool stop);
 	bool DoQuit(bool quit);
+	bool ClearConnections(bool clearconnections);
 	bool TrackStats();
 	
 	bool Finalise();
@@ -83,8 +143,8 @@ class ReceiveSQL{
 	bool UpdateRole();
 	boost::posix_time::ptime ToTimestamp(const std::string& timestring);
 	std::string ToTimestring(boost::posix_time::ptime);
-	bool TimeStringFromUnixMs(uint64_t timestamp, std::string& timestring);
-	bool TimeStringFromUnixSec(uint64_t timestamp, std::string& timestring);
+	bool TimeStringFromUnixMs(const uint64_t& timestamp, std::string& timestring);
+	bool TimeStringFromUnixSec(const uint64_t& timestamp, std::string& timestring);
 	bool GetLastUpdateTime(std::string& our_timestamp);
 	
 	// Logging functions
@@ -116,6 +176,10 @@ class ReceiveSQL{
 	zmq::socket_t* clt_rtr_socket=nullptr;  // receives read queries from client dealers
 	zmq::socket_t* clt_sub_socket=nullptr;  // receives write queries from client publishers
 	zmq::socket_t* mm_rcv_socket=nullptr;   // receives connections from other middlemen
+	// mutexes for locking them while background service discovery thread makes new connections
+	std::mutex clt_rtr_mtx;
+	std::mutex clt_sub_mtx;
+	std::mutex mm_rcv_mtx;
 	
 	// these sockets will bind, they advertise our services
 	zmq::socket_t* mm_snd_socket=nullptr;   // we will advertise our presence as a middleman to other middlemen
@@ -124,20 +188,15 @@ class ReceiveSQL{
 	// and connect us to those sockets
 	ServiceDiscovery* service_discovery = nullptr;
 	MMUtilities* utilities = nullptr;
-	// required by the Utilities class to keep track of connections to clients
-	// we should have one map per zmq_socket managed by the Utilities class;
-	// it uses this to determine if we are connected to a given client already
-	std::map<std::string,Store*> clt_rtr_connections;
-	std::map<std::string,Store*> mm_rcv_connections;
-	std::map<std::string,Store*> clt_sub_connections;
+	
+	// for background thread creation
+	Utilities m_util;
 	
 	// multicast socket file descriptor
 	int multicast_socket=-1;
 	// multicast destination address structure
 	struct sockaddr_in multicast_addr;
 	socklen_t multicast_addrlen;
-	// apparently works with zmq poller?
-	zmq::pollitem_t multicast_poller;
 	
 	// poll timeouts
 	int inpoll_timeout;
@@ -190,7 +249,9 @@ class ReceiveSQL{
 	std::map<std::pair<std::string, uint32_t>, Query> wrt_txn_queue;
 	std::map<std::pair<std::string, uint32_t>, Query> rd_txn_queue;
 	std::map<std::pair<std::string, uint32_t>, Query> resp_queue;
-	std::deque<std::string> in_multicast_queue;
+	std::vector<std::string> in_multicast_queue;
+	std::mutex in_multicast_queue_mtx;
+	int multicasts_waiting; // how many multicasts the thread has yet to write to DB
 	std::deque<std::string> out_multicast_queue;
 	// we'll cache a set of recent responses send to each client,
 	// then if a client that misses their acknowledgement and resends the query,
diff --git a/ReceiveSQLConfig b/ReceiveSQLConfig
index cfbb7ca..848aa02 100644
--- a/ReceiveSQLConfig
+++ b/ReceiveSQLConfig
@@ -10,7 +10,7 @@ port 5432
 dbname daq
 #user tooldaq
 #passwd tooldaq19
-#context_io_threads 1
+context_io_threads 3
 clt_sub_port 55556
 clt_rtr_port 55555
 mm_snd_port 55597
@@ -18,6 +18,7 @@ log_pub_port 55554
 remote_control_port 24011
 multicast_port 55554
 multicast_address 239.192.1.1
+num_multicast_threads 10
 
 # all timeouts in ms
 clt_sub_socket_timeout 500
@@ -42,7 +43,7 @@ max_send_attempts 3
 # how many postgres queries in the queue before we warn about being full
 warn_limit 700
 # how many postgres queries before we start dropping them
-drop_limit 1000
+drop_limit 5000
 # how long to cache received messages so we don't get duplicates
 cache_period_ms 10000
 # if writes are sent to the read-only port, but we're master, should we accept them?
diff --git a/run_maintenance.sh b/run_maintenance.sh
new file mode 100755
index 0000000..3d00d73
--- /dev/null
+++ b/run_maintenance.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+set -x
+while [ /bin/true ]; do
+	psql -c "CALL partman.partition_data_proc('public.monitoring', p_batch := 1);"
+	NDEFAULT=$(psql -At -c "SELECT * FROM partman.check_default();"  | cut -d'|' -f 2 );
+	#if [ $? -ne 0 ]; then
+	if [ ! -z "${NDEFAULT}" ] && [ ${NDEFAULT} -eq 0 ]; then
+		break;
+	else
+		sleep 1200
+	fi
+done
+sleep 1200
+#psql -c "VACUUM ANALYZE monitoring;"

From 316a288f9b9f9aa9e7d21b70eac1369837a513cc Mon Sep 17 00:00:00 2001
From: Marcus O'Flaherty <marcus.o-flaherty@warwick.ac.uk>
Date: Thu, 5 Jun 2025 15:50:33 +0000
Subject: [PATCH 4/7] add folder for testing apps, Makefile needs merging

---
 MessageMiddleman/Makefile              |  25 ++++++
 MessageMiddleman/MessageMiddleman.cpp  |  61 +++++++++++++++
 MessageMiddleman/MiddlemanTesterConfig |  17 +++++
 MessageMiddleman/ReceiveAlert.cpp      | 102 +++++++++++++++++++++++++
 MessageMiddleman/SendAlert.cpp         |  25 ++++++
 5 files changed, 230 insertions(+)
 create mode 100644 MessageMiddleman/Makefile
 create mode 100644 MessageMiddleman/MessageMiddleman.cpp
 create mode 100644 MessageMiddleman/MiddlemanTesterConfig
 create mode 100644 MessageMiddleman/ReceiveAlert.cpp
 create mode 100644 MessageMiddleman/SendAlert.cpp

diff --git a/MessageMiddleman/Makefile b/MessageMiddleman/Makefile
new file mode 100644
index 0000000..9412078
--- /dev/null
+++ b/MessageMiddleman/Makefile
@@ -0,0 +1,25 @@
+# minimal middleman tester
+Dependencies=/opt
+
+ZMQLib= -L $(Dependencies)/zeromq-4.0.7/lib -lzmq
+ZMQInclude= -I $(Dependencies)/zeromq-4.0.7/include/
+
+BoostLib= -L $(Dependencies)/boost_1_66_0/install/lib -lboost_date_time -lboost_serialization -lboost_iostreams
+BoostInclude= -I $(Dependencies)/boost_1_66_0/install/include
+
+ToolDAQLib= -L $(Dependencies)/ToolDAQFramework/lib   -lDAQDataModelBase -lServiceDiscovery -lDAQStore
+ToolDAQInclude= -I $(Dependencies)/ToolDAQFramework/include
+
+ToolFrameworkLib= -L $(Dependencies)/ToolFrameworkCore/lib -lStore -lDataModelBase
+ToolFrameworkInclude= -I $(Dependencies)/ToolFrameworkCore/include
+
+all: MessageMiddleman SendAlert ReceiveAlert
+
+MessageMiddleman: MessageMiddleman.cpp lib/libDAQInterface.so
+        g++ -O3  -Wpedantic -std=c++11 $< -o $@ -I ./include/ -L lib/ -lDAQInterface -lpthread $(ToolDAQInclude) $(ToolDAQLib) $(ToolFrameworkInclude) $(ToolFrameworkLib) $(BoostInclude) $(ZMQInclude) $(ZMQLib) $(ToolDAQLib) $(BoostLib) $(ToolDAQLib)
+
+SendAlert: SendAlert.cpp lib/libDAQInterface.so
+        g++ -O3  -Wpedantic -std=c++11 $< -o $@ -I ./include/ -L lib/ -lDAQInterface -lpthread $(ToolDAQInclude) $(ToolDAQLib) $(ToolFrameworkInclude) $(ToolFrameworkLib) $(BoostInclude) $(ZMQInclude) $(ZMQLib) $(ToolDAQLib) $(BoostLib) $(ToolDAQLib)
+        
+ReceiveAlert: ReceiveAlert.cpp lib/libDAQInterface.so
+        g++ -O3  -Wpedantic -std=c++11 $< -o $@ -I ./include/ -L lib/ -lDAQInterface -lpthread $(ToolDAQInclude) $(ToolDAQLib) $(ToolFrameworkInclude) $(ToolFrameworkLib) $(BoostInclude) $(ZMQInclude) $(ZMQLib) $(ToolDAQLib) $(BoostLib) $(ToolDAQLib)
diff --git a/MessageMiddleman/MessageMiddleman.cpp b/MessageMiddleman/MessageMiddleman.cpp
new file mode 100644
index 0000000..38b3efd
--- /dev/null
+++ b/MessageMiddleman/MessageMiddleman.cpp
@@ -0,0 +1,61 @@
+#include <iostream>
+#include <locale>
+#include <DAQInterface.h>
+
+using namespace ToolFramework;
+
+int main(int argc, const char* argv[]){
+	
+	if(argc<3){
+		std::cout<<"usage: "<<argv[0]<<" [type] [message]"<<std::endl;
+		std::cout<<"[type]: 0=multicast, 1=tcp"<<std::endl;
+		return 0;
+	}
+	
+	std::string topic = argv[1];
+	std::string msg="";
+	for(int i=2; i<argc; ++i){
+		msg += std::string(argv[i]);
+	}
+	for(char& c : topic){
+		c=std::tolower(c);
+	}
+	
+	std::string Interface_configfile = "./MiddlemanTesterConfig";
+	std::cout<<"Constructing DAQInterface (will incur a wait to allow middleman to connect)..."<<std::endl;
+	DAQInterface DAQ_inter(Interface_configfile);
+	std::cout<<"Constructed"<<std::endl;
+	DAQ_inter.sc_vars["Status"]->SetValue("Initialising"); //setting status message
+	
+	if(topic=="log"){
+		// send logging multicast message
+		std::cout<<"logging '"<<msg<<"'"<<std::endl;
+		bool ok = DAQ_inter.SendLog(msg);
+		std::cout<<"ok: "<<ok<<std::endl;
+	} else if(topic=="monitor"){
+		// send monitoring multicast message
+		std::cout<<"monitoring '"<<msg<<"'"<<std::endl;
+		bool ok = DAQ_inter.SendMonitoringData(msg);
+		std::cout<<"ok: "<<ok<<std::endl;
+	} else if(topic=="alarm"){
+		// send tcp message
+		std::cout<<"sending alarm '"<<msg<<"'"<<std::endl;
+		bool ok = DAQ_inter.SendAlarm(msg);
+		std::cout<<"ok: "<<ok<<std::endl;
+	} else if(topic=="query"){
+		std::string resp;
+		std::cout<<"sending query '"<<msg<<"'"<<std::endl;
+		bool ok = DAQ_inter.SQLQuery("daq",msg,resp, 1000);
+		std::cout<<"ok: "<<ok<<", resp: "<<resp<<std::endl;
+	} else if(topic=="rconfig"){
+		std::string resp;
+		std::cout<<"requesting config for device "<<msg<<std::endl;
+		bool ok = DAQ_inter.GetDeviceConfig(resp, -1, msg);
+		std::cout<<"ok: "<<ok<<", resp: "<<resp<<std::endl;
+	} else {
+		std::cerr<<"unknown topic"<<std::endl;
+	}
+	
+	return 0;
+	
+}
diff --git a/MessageMiddleman/MiddlemanTesterConfig b/MessageMiddleman/MiddlemanTesterConfig
new file mode 100644
index 0000000..f7f0a00
--- /dev/null
+++ b/MessageMiddleman/MiddlemanTesterConfig
@@ -0,0 +1,17 @@
+UUID b16a0186-c5df-474c-b880-9fc6c0670d7d  # unique identifier for client (must be valid uuid)
+device_name MMTest 			# name of devices
+verbosity 1 				# Verbosity level of interface
+db_name daq                     	#
+max_retries 3				#	
+advertise_endpoints 0			#
+resend_period_ms 1000			#
+print_stats_period_ms 1000		#
+clt_pub_port 55556    			#
+clt_dlr_port 55555			#
+clt_pub_socket_timeout 500		#	
+clt_dlr_socket_timeout 500		#
+inpoll_timeout 50      			# keep these short!
+outpoll_timeout 50 	 		# keep these short!
+command_timeout 2000			#
+multicast_port 55554			#
+multicast_address 239.192.1.1		#
diff --git a/MessageMiddleman/ReceiveAlert.cpp b/MessageMiddleman/ReceiveAlert.cpp
new file mode 100644
index 0000000..ef10d84
--- /dev/null
+++ b/MessageMiddleman/ReceiveAlert.cpp
@@ -0,0 +1,102 @@
+#include <iostream>
+#include <fstream>
+#include <signal.h>
+#include <DAQInterface.h>
+#include <chrono>
+
+using namespace ToolFramework;
+
+bool running = false;
+void stopSignalHandler(int _ignored){
+  running = false;
+}
+
+std::ofstream outfile;
+
+// class for automating functions from slowcontrol
+class AutomatedFunctions {
+  
+  public:
+  AutomatedFunctions(DAQInterface* in_DAQ_inter){
+    DAQ_inter=in_DAQ_inter;
+  };
+  
+  DAQInterface* DAQ_inter;
+  void TestAlert_func(const char* alert, const char* payload){
+    time_t now = std::chrono::high_resolution_clock::to_time_t(std::chrono::high_resolution_clock::now());
+    std::string payloadstr = "";
+    if(payload!=nullptr) payloadstr = payload;
+    std::cout<<"recevied a '"<<alert<<"' alert with payload '"<<payloadstr<<"' at "<<ctime(&now)<<std::endl;
+    outfile<<"recevied a '"<<alert<<"' alert with payload '"<<payloadstr<<"' at "<<ctime(&now)<<std::endl;
+  }
+  
+};
+
+int main(){
+
+  // listen for SIGINT
+  signal((int) SIGINT, stopSignalHandler);
+  
+  outfile.open("alertlogs.txt");
+  
+  int verbose=1;
+  
+  ////////////////////////////// setup /////////////////////////////////
+  
+  std::string Interface_configfile = "./InterfaceConfig";
+  
+  std::cout<<"Constructing DAQInterface"<<std::endl;
+  DAQInterface DAQ_inter(Interface_configfile);
+  std::string device_name = DAQ_inter.GetDeviceName(); //name of my device
+  
+  std::cout<<"Constructing an AutomatedFunctions helper class to encapsulate callback functions"<<std::endl;
+  AutomatedFunctions automated_functions(&DAQ_inter);
+  // N.B. callback functions can simply be ordinary free functions, or they may be member functions
+  // as in this case, where we define an 'AutomatedFunctions' helper class to hold them.
+  // The advantage of using a helper class such as this is that the helper class can keep a handle
+  // to the DAQInterface instance, which callback functions may then use to send logs, alarms,
+  // update the service status, and so on.
+  
+  //////////////////////////////////////////////////////////////////////
+  
+  std::cout<<"Updating service status to 'Initialising'"<<std::endl;
+  DAQ_inter.sc_vars["Status"]->SetValue("Initialising"); //setting status message
+  
+  std::cout<<"Registering callback function 'AutomatedFunctions::TestAlert_func' to be invoked on alert 'TestAlert'..."<<std::flush;
+  // if the DAQ sends out a global "TestAlert" alert, the registered callback function will be automatically invoked
+  std::vector<std::string> alertnames{"TestAlert","RunStart","ChangeConfig","RunStop","LEDTrigger","SoftTrigger"};
+  for(std::string& alert_name : alertnames){
+    DAQ_inter.AlertSubscribe(alert_name,  std::bind(&AutomatedFunctions::TestAlert_func, automated_functions,  std::placeholders::_1, std::placeholders::_2));
+  }
+  std::cout<<"Done"<<std::endl;
+  
+  ///////////////////////////////////////////////////////////////////////////
+  
+  std::cout<<"Updating service status to 'Ready'"<<std::endl;
+  DAQ_inter.sc_vars["Status"]->SetValue("Ready");
+   
+  if(verbose) std::cout<<"Registering 'Quit' button..."<<std::flush;
+  DAQ_inter.sc_vars.Add("Quit",BUTTON);
+  DAQ_inter.sc_vars["Quit"]->SetValue(false);
+  if(verbose) std::cout<<"Done"<<std::endl;
+  
+  running=true;
+  
+  while(running){ // run until user clicks 'Quit' slow control
+       
+    // check if the quit button has been pressed
+    running=(!DAQ_inter.sc_vars["Quit"]->GetValue<bool>()); 
+    
+    usleep(1000);
+    
+  } // end of program loop
+  
+  
+  std::cout<<"Application terminated"<<std::endl;
+  DAQ_inter.sc_vars["Status"]->SetValue("Terminated");
+  
+  outfile.close();
+  
+  return 0;
+  
+}
diff --git a/MessageMiddleman/SendAlert.cpp b/MessageMiddleman/SendAlert.cpp
new file mode 100644
index 0000000..b5528c9
--- /dev/null
+++ b/MessageMiddleman/SendAlert.cpp
@@ -0,0 +1,25 @@
+#include <iostream>
+#include <DAQInterface.h>
+
+using namespace ToolFramework;
+
+int main(){
+
+  int verbose=1;
+  
+  ////////////////////////////// setup /////////////////////////////////
+  
+  std::string Interface_configfile = "./InterfaceConfig";
+  //std::string database_name = "daq";
+  
+  std::cout<<"Constructing DAQInterface"<<std::endl;
+  DAQInterface DAQ_inter(Interface_configfile);
+  std::string device_name = DAQ_inter.GetDeviceName(); //name of my device
+  
+  std::cout<<"sending test 'TestAlert' alert"<<std::endl;
+  bool ok = DAQ_inter.AlertSend("TestAlert","TEST!");
+  std::cout<<"send ok: "<<ok<<std::endl;
+
+  return 0;
+  
+}

From d36f8dcb6e7b74efd55abe157c87737a3cfbc307 Mon Sep 17 00:00:00 2001
From: Marcus O'Flaherty <moflaherty1@sheffield.ac.uk>
Date: Fri, 21 Nov 2025 13:22:41 +0000
Subject: [PATCH 5/7] bugfixes, seems to be working

---
 ReceiveSQL.cpp | 54 +++++++++++++++++++++++++++++++++++---------------
 ReceiveSQL.h   |  7 +++++--
 Setup.sh       |  2 ++
 3 files changed, 45 insertions(+), 18 deletions(-)

diff --git a/ReceiveSQL.cpp b/ReceiveSQL.cpp
index 2d4fd36..d5a1443 100755
--- a/ReceiveSQL.cpp
+++ b/ReceiveSQL.cpp
@@ -290,8 +290,12 @@ bool ReceiveSQL::Finalise(){
 	Log("Waiting for multicast workers to finish...",3);
 	for(int i=0; i<secs_to_wait; ++i){
 		bool all_done=true;
-		for(MulticastWorker_args* thread_args : multicast_worker_args){
-			if(!thread_args->finished) all_done=false;
+		for(int j=0; j<multicast_worker_args.size(); ++j){
+			MulticastWorker_args* thread_args = multicast_worker_args.at(j);
+			if(!thread_args->finished){
+				all_done=false;
+				break;
+			}
 		}
 		if(all_done) break;
 		std::this_thread::sleep_for(std::chrono::seconds(1));
@@ -407,6 +411,8 @@ bool ReceiveSQL::InitMulticast(Store& m_variables){
 	m_variables.Get("multicast_address",multicast_address);
 	int num_multicast_threads=10;
 	m_variables.Get("num_multicast_threads",num_multicast_threads);
+	m_variables.Get("batch_size",batch_size);
+	m_variables.Get("max_hold_ms",max_hold_ms);
 	
 	// set up multicast socket for sending logging & monitoring data
 	log_socket = socket(AF_INET, SOCK_DGRAM, 0);
@@ -464,7 +470,7 @@ bool ReceiveSQL::InitMulticast(Store& m_variables){
 	
 	// used in send & receive; will be the same for both log & mon
 	multicast_addrlen = sizeof(log_addr);
-
+	
 	/*
 	// for two-way comms, we should bind to INADDR_ANY, not a specific multicast address.... maybe?
 	struct sockaddr_in multicast_addr2;
@@ -498,7 +504,9 @@ bool ReceiveSQL::InitMulticast(Store& m_variables){
 	}
 	
 	multicast_listener_args = new MulticastReceive_args;
+	multicast_listener_args->type=multicast_type::log;
 	multicast_listener_args->parent = this;
+	multicast_listener_args->running = true;
 	multicast_listener_args->finished = false;
 	multicast_listener_args->socket = log_socket;
 	multicast_listener_args->addr = &log_addr;
@@ -507,7 +515,9 @@ bool ReceiveSQL::InitMulticast(Store& m_variables){
 	m_util.CreateThread("log_receiver", &GetMulticastMessages, multicast_listener_args);
 	
 	multicast_listener_args = new MulticastReceive_args;
+	multicast_listener_args->type=multicast_type::mon;
 	multicast_listener_args->parent = this;
+	multicast_listener_args->running = true;
 	multicast_listener_args->finished = false;
 	multicast_listener_args->socket = mon_socket;
 	multicast_listener_args->addr = &mon_addr;
@@ -526,8 +536,9 @@ bool ReceiveSQL::InitMulticast(Store& m_variables){
 			Log("Failed to open connection to database for log worker thread!",v_error);
 			return false;
 		}
-		MulticastWorker_args* thread_args=new MulticastWorker_args(this, conn, multicast_type::log, &in_log_queue, &in_log_queue_mtx);
-		m_util.CreateThread("log_worker", &MulticastWorker, thread_args);
+		MulticastWorker_args* thread_args=new MulticastWorker_args(this, conn, multicast_type::log, &in_log_queue, &in_log_queue_mtx, i);
+		std::string name="log_worker_"+std::to_string(i); // like ROOT histograms, they do need a unique name
+		m_util.CreateThread(name, &MulticastWorker, thread_args);
 		multicast_worker_args.push_back(thread_args);
 	}
 	
@@ -537,8 +548,9 @@ bool ReceiveSQL::InitMulticast(Store& m_variables){
 			Log("Failed to open connection to database for multicast thread!",v_error);
 			return false;
 		}
-		MulticastWorker_args* thread_args=new MulticastWorker_args(this, conn, multicast_type::mon, &in_mon_queue, &in_mon_queue_mtx);
-		m_util.CreateThread("mon_worker", &MulticastWorker, thread_args);
+		MulticastWorker_args* thread_args=new MulticastWorker_args(this, conn, multicast_type::mon, &in_mon_queue, &in_mon_queue_mtx, i+(num_multicast_threads/2));
+		std::string name="mon_worker_"+std::to_string(i);
+		m_util.CreateThread(name, &MulticastWorker, thread_args);
 		multicast_worker_args.push_back(thread_args);
 	}
 	
@@ -1869,16 +1881,20 @@ void ReceiveSQL::GetMulticastMessages(Thread_args* arg){
 	MulticastReceive_args* m_args=reinterpret_cast<MulticastReceive_args*>(arg);
 	ReceiveSQL& p = *m_args->parent;
 	
-	if(m_args->finished) return;
+	if(m_args->finished){
+		std::this_thread::sleep_for(std::chrono::milliseconds(100)); // throttler.... probably unneeded
+		return;
+	}
 	
 	// check if parent has told us to finalise
 	if(!m_args->running){
 		// parent has told us to finalise
 		m_args->finished=true;
+		return;
 	}
 	
 	// poll multicast socket
-	zmq::poll(&m_args->poll, 1, 100);
+	zmq::poll(&m_args->poll, 1, 500);
 	
 	// see if we had any multicast messages
 	if(m_args->poll.revents & ZMQ_POLLIN){
@@ -1906,12 +1922,12 @@ void ReceiveSQL::GetMulticastMessages(Thread_args* arg){
 			p.Log("Put multicast msg in queue: '"+std::string(m_args->message)+"'",12);
 			
 		}
-	} /*else { std::cout<<"no multicast messages"<<std::endl; }*/
+	} /*else { std::cout<<"no multicast messages"<<std::endl; } */
 	
 	p.Log("Trimming In Multicast Deque",20);
 	// FIXME don't do this here? Or... is there a better place?
-	m_args->get_ok = p.TrimVector("in_log_queue");
-	m_args->get_ok = p.TrimVector("in_mon_queue");
+	if(m_args->type==multicast_type::log) m_args->get_ok = p.TrimVector("in_log_queue");
+	else m_args->get_ok = p.TrimVector("in_mon_queue");
 	
 	return;
 }
@@ -2332,7 +2348,10 @@ void ReceiveSQL::MulticastWorker(Thread_args* arg){
 	MulticastWorker_args* m_args=reinterpret_cast<MulticastWorker_args*>(arg);
 	ReceiveSQL& p = *m_args->parent;
 	
-	if(m_args->finished) return;
+	if(m_args->finished){
+		std::this_thread::sleep_for(std::chrono::milliseconds(100)); // throttler.... probably unneeded
+		return;
+	}
 	
 	// check if parent has new messages for us
 	m_args->in_queue_mtx->lock();
@@ -2344,7 +2363,8 @@ void ReceiveSQL::MulticastWorker(Thread_args* arg){
 			m_args->finished=true;
 		}
 		
-		usleep(100);  // don't keep locking it too often
+		// don't keep locking it too often
+		std::this_thread::sleep_for(std::chrono::microseconds(100));
 		return;
 	}
 	
@@ -2367,7 +2387,8 @@ void ReceiveSQL::MulticastWorker(Thread_args* arg){
 			m_args->query += m_args->vals_string;
 			m_args->first_vals = false;
 			++m_args->n_queries;
-		}
+			// FIXME errors should never silently disappear
+		} /*else { printf("MC parse error\n"); } */
 		
 	}
 	m_args->msg_queue.resize(0);
@@ -2375,7 +2396,7 @@ void ReceiveSQL::MulticastWorker(Thread_args* arg){
 	// see if we have enough messages for a transaction,
 	// or it's been long enough to force one to prevent messages sitting too long without insertion
 	boost::posix_time::time_duration lapse = m_args->last_insert - boost::posix_time::microsec_clock::universal_time();
-	if(m_args->n_queries < p.batch_size && lapse.total_seconds() < 30) return;
+	if(m_args->n_queries < p.batch_size && lapse.total_milliseconds() < p.max_hold_ms) return;
 	
 	// merge all messages and do batch insertion
 	m_args->query += ";";
@@ -2396,6 +2417,7 @@ void ReceiveSQL::MulticastWorker(Thread_args* arg){
 	if(not m_args->get_ok){
 		// something went wrong
 		// can't use Log or we end up in a circular loop since Log is a multicast query itself.
+		// FIXME printouts for error not great
 		std::cerr<<"Error inserting logmessage '"<<m_args->query<<"' into database"<<std::endl;
 		if(m_args->type==multicast_type::log) p.log_queries_failed+=m_args->n_queries;
 		else p.mon_queries_failed+=m_args->n_queries;
diff --git a/ReceiveSQL.h b/ReceiveSQL.h
index 7d76da9..4773d17 100755
--- a/ReceiveSQL.h
+++ b/ReceiveSQL.h
@@ -75,7 +75,8 @@ class ReceiveSQL{
 	std::vector<MulticastWorker_args*> multicast_worker_args;
 	MulticastReceive_args* multicast_listener_args=nullptr;
 	FindClients_args* findclients_args=nullptr;
-	size_t batch_size = 100; // minimum number of multicast messages to buffer before insert (or >30s)
+	size_t batch_size = 100;  // minimum number of multicast messages to buffer before insert
+	size_t max_hold_ms = 500; // or wait at most this long before inserting any new multicast messages
 	bool SendNextReply();
 	bool SendNextMulticast();
 	std::string escape_json(std::string s);
@@ -462,6 +463,7 @@ struct FindClients_args : public Thread_args {
 
 // POD class for things passed to multicast worker threads
 struct MulticastWorker_args : public Thread_args {
+	short id;
 	ReceiveSQL* parent;
 	bool running;
 	bool finished;
@@ -481,7 +483,8 @@ struct MulticastWorker_args : public Thread_args {
 	std::mutex* in_queue_mtx;
 	multicast_type type;
 	
-	MulticastWorker_args(ReceiveSQL* i_parent, pqxx::connection* i_conn, multicast_type worker_type, std::vector<std::string>* msg_queue, std::mutex* msg_queue_mtx){
+	MulticastWorker_args(ReceiveSQL* i_parent, pqxx::connection* i_conn, multicast_type worker_type, std::vector<std::string>* msg_queue, std::mutex* msg_queue_mtx, short iid){
+		id=iid;
 		parent = i_parent;
 		conn = i_conn;
 		type = worker_type;
diff --git a/Setup.sh b/Setup.sh
index f127a21..a66967f 100755
--- a/Setup.sh
+++ b/Setup.sh
@@ -1,5 +1,7 @@
 #!/bin/bash
 Dependencies=/opt
+#export PS1='${debian_chroot:+($debian_chroot)}\[\033[35;2m\]\u@\h\[\033[02;31;22m\][\t]\[\033[00m\]:\[\033[00;36m\]\w\[\033[00m\]\$ '
+export PS1='${debian_chroot:+($debian_chroot)}\[\033[35;2;1m\]\u@\h\[\033[00m\]:\[\033[00;36m\]\w\[\033[00m\]\$ '
 
 # add Dependencies from ToolFramework
 export LD_LIBRARY_PATH=${Dependencies}/zeromq-4.0.7/lib:${Dependencies}/boost_1_66_0/install/lib:${Dependencies}/libpqxx-6.4.5/install/lib:${Dependencies}/ToolFrameworkCore/lib:${Dependencies}/ToolDAQFramework/lib:$LD_LIBRARY_PATH

From 6e60afc7c33015a6b391d111a45deb1059bf80ff Mon Sep 17 00:00:00 2001
From: Marcus O'Flaherty <moflaherty1@sheffield.ac.uk>
Date: Tue, 2 Dec 2025 14:54:38 +0000
Subject: [PATCH 6/7] fix edge case parsing in JsonParser

---
 JsonParser.cpp |  50 +++++++++++---
 JsonTester.cpp | 175 +++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 216 insertions(+), 9 deletions(-)
 create mode 100644 JsonTester.cpp

diff --git a/JsonParser.cpp b/JsonParser.cpp
index a944309..53b3f7a 100644
--- a/JsonParser.cpp
+++ b/JsonParser.cpp
@@ -132,6 +132,12 @@ bool JSONP::ScanJsonArray(const std::string& thejson, JsonParserResult& result){
 		return true;
 	}
 	
+	// empty array
+	if(thejson.find_first_not_of(" ,\n")==std::string::npos){
+		result.type=JsonParserResultType::empty;
+		return true;
+	}
+	
 	// json arrays are annoyingly flexible; they can be homogeneous,
 	// containing ints, floats, strings, bools, nulls or objects,
 	// but they can also be inhomogeneous combining elements of all these different types
@@ -183,22 +189,22 @@ bool JSONP::ScanJsonArray(const std::string& thejson, JsonParserResult& result){
 		// could be floats or strings, or arrays or objects
 	}
 	// rule out integers by anything other than digits and signs
-	if(thejson.find_first_not_of("01234567890+-, ")!=std::string::npos){
+	if(thejson.find_first_not_of("01234567890+-, \n")!=std::string::npos){
 		if(verbose) std::cout<<"found something other than digits: can't be ints"<<std::endl;
 		all_sints=false;
 		all_uints=false;
 	}
 	// rule out doubles by anything other than numbers, signs and scientific notation characters
-	if(thejson.find_first_not_of("0123456789+-.Ee^*, ")!=std::string::npos){
+	if(thejson.find_first_not_of("0123456789+-.Ee^*, \n")!=std::string::npos){
 		if(verbose) std::cout<<"found something other than SI characters: can't be floats"<<std::endl;
 		all_floats=false;
 	}
 	// rule out bools and nulls by finding anything other than the corresponding characters
-	if(thejson.find_first_not_of("tTrRuUeEfFaAlLsSeE, ")!=std::string::npos){
+	if(thejson.find_first_not_of("tTrRuUeEfFaAlLsSeE, \n")!=std::string::npos){
 		if(verbose) std::cout<<"found characters not in true or false; can't be bools"<<std::endl;
 		all_bools=false;
 	}
-	if(thejson.find_first_not_of("nNuUlL, ")!=std::string::npos){
+	if(thejson.find_first_not_of("nNuUlL, \n")!=std::string::npos){
 		if(verbose) std::cout<<"found characters not in null; can't be nulls"<<std::endl;
 		all_nulls=false;
 	}
@@ -233,8 +239,9 @@ bool JSONP::ScanJsonArray(const std::string& thejson, JsonParserResult& result){
 		// note that as entry elements may be objects, nested arrays, or strings that contain commas,
 		// we can't just treat it as a comma-delimited list
 		bool in_string=false;
+		bool escaped=false;
 		std::vector<char> delimiters;
-		if(verbose)	std::cout<<"scanning remaining string: "
+		if(verbose) std::cout<<"scanning remaining string: "
 		                     <<thejson.substr(next_start,std::string::npos)<<std::endl;
 		for(next_end=next_start; next_end<thejson.length(); ++next_end){
 			if(verbose){
@@ -244,9 +251,17 @@ bool JSONP::ScanJsonArray(const std::string& thejson, JsonParserResult& result){
 					if(k>0) std::cout<<", ";
 					std::cout<<delimiters.at(k);
 				}
-				std::cout<<std::endl;
+				std::cout<<", escaped: "<<escaped<<std::endl;
 			}
 			char nextchar = thejson.at(next_end);
+			if(escaped){
+				escaped=false;
+				continue;
+			}
+			if(in_string && nextchar=='\\'){
+				escaped=true;
+				continue;
+			}
 			if(in_string && nextchar=='"'){
 				in_string=false;
 				continue;
@@ -273,7 +288,8 @@ bool JSONP::ScanJsonArray(const std::string& thejson, JsonParserResult& result){
 		if(verbose) std::cout<<"next array element is "<<tmp<<std::endl;
 		tmp=Trim(tmp);
 		if(verbose) std::cout<<"trimmed is '"<<tmp<<"'"<<std::endl;
-		
+		// ignore empty elements?
+		if(tmp.empty()) continue;
 		if(tmp.front()=='{' || tmp.front()=='['){
 			// found an object or array
 			// since inhomogeneous element types are valid, we could have already parsed
@@ -606,7 +622,7 @@ bool JSONP::ScanJsonPrimitive(std::string thejson, std::string thekey, BStore& o
 	
 	try {
 		if(verbose) std::cout<<"try int"<<std::endl;
-
+		
 		// discard leading whitespace
 		size_t startpos=0;
 		size_t endpos=0;
@@ -683,6 +699,11 @@ bool JSONP::ScanJsonObject(std::string thejson, BStore& outstore){
 	// should be sequence of comma delimited key-value pairs,
 	// with string keys separated from values by colons
 	
+	// technically JSON objects and arrays may have a trailing comma
+	thejson=Trim(thejson);
+	if(thejson.back()==',') thejson.pop_back();
+	thejson=Trim(thejson);
+	
 	// trivial case
 	if(thejson=="") return true;
 	
@@ -691,6 +712,7 @@ bool JSONP::ScanJsonObject(std::string thejson, BStore& outstore){
 	size_t next_end=std::string::npos;
 	bool key=true; // alternate key and value
 	std::string next_key;
+	bool escaped=false;
 	while(true){
 		// find the end of the next array entry
 		// note that as entry elements may be objects, nested arrays, or strings that contain commas,
@@ -707,9 +729,17 @@ bool JSONP::ScanJsonObject(std::string thejson, BStore& outstore){
 					if(k>0) std::cout<<", ";
 					std::cout<<delimiters.at(k);
 				}
-				std::cout<<std::endl;
+				std::cout<<", escaped: "<<escaped<<std::endl;
 			}
 			char& nextchar = thejson.at(next_end);
+			if(escaped){
+				escaped=false;
+				continue;
+			}
+			if(in_string && nextchar=='\\'){
+				escaped=true;
+				continue;
+			}
 			if(in_string && nextchar=='"'){
 				in_string=false;
 				if(key && delimiters.empty()){
@@ -753,6 +783,8 @@ bool JSONP::ScanJsonObject(std::string thejson, BStore& outstore){
 		
 		// if processing a key record it and continue to next loop
 		if(key){
+			// ignore duplicated delimiters
+			if(tmp.empty()) continue;
 			next_key=tmp;
 			if(verbose) std::cout<<"it key"<<std::endl;
 			// sanity checks, key should be a string
diff --git a/JsonTester.cpp b/JsonTester.cpp
new file mode 100644
index 0000000..c46c54a
--- /dev/null
+++ b/JsonTester.cpp
@@ -0,0 +1,175 @@
+#include "JsonParser.h"
+#include "BoostStore.h"
+#include <iostream>
+
+int main(int argc, const char** argv){
+//	std::string testjson="{\"not true\": [0, false], \"true\": true, \"not null\": [0, 1, false, true, {\"obj\": null}, \"a string\"] }";
+//	std::string testjson="{}";
+//	std::string testjson="[0]";
+//	std::string testjson="{\"Sally\": \"Father McKenzie, I thought you were DEAD!\", \"McKenzie\": \"I vas!\"}";
+//	std::string testjson="[{   \"why\":null} ]";
+//	std::string testjson="{ \"xvals\":[\"2022-09-23 02:04:55\",\"2022-09-23 01:48:08\",\"2022-09-23 01:31:45\"], \"yvals\":[0.497962,0.502168,0.498470] }";
+//	std::string testjson="{\"purefit\":1, \"fits\":[ {\"method\":\"raw\", \"absfit\":1, \"peakdiff\":0.135514, \"gdconc\":0.055772 }, {\"method\":\"simple\", \"absfit\":1, \"peakdiff\":0.118749, \"gdconc\":0.055036 }, {\"method\":\"complex\", \"absfit\":1, \"peakdiff\":0.120748, \"gdconc\":0.055113 } ] }";
+//	std::string testjson="{\"runnum\":128,\"start\":\"2022-09-09T00:21:09.356056\",\"stop\":null,\"runconfig\":1,\"notes\":\"\nfirst in-situ run with database, marcusanalysis, transparency and web page updating.\",\"git_tag\":\"V2.1.2-76-g2b4c532\",  \"output_file\": \"05SeptEGADS\" }";
+//	std::string testjson="[ 1, 2, 3.4 ]";  // returns std::vector<double>
+// all the following convert to std::vector<BoostStore>
+//	std::string testjson="[ 1, true ]";
+//	std::string testjson="[ 1, 1.1, true ]";
+//	std::string testjson="[ 1, 2.2, \"three\"]";
+//	std::string testjson="[ 1, null ]";
+//	std::string testjson="[ 1, true, null ]";
+//	std::string testjson="[ 1, true, null, \"potato\" ]";
+//	std::string testjson="[ [1,2,3], [4.4,5.5,6.6] ]";
+//	std::string testjson="{\"VME\": 1, \"PGTool\": 2, \"SlackBot\": 3, \"RunControl\": 4}";
+	
+//	// adapted from https://github.com/briandfoy/json-acceptance-tests, pass1
+	std::string testjson=
+		"{\n"
+//		"\"integer\": 1234567890,\n"
+//		"\"real\": -9876.543210,\n"
+//		"\"e\": 0.123456789e-12,\n"
+//		"\"E\": 1.234567890E+34,\n"
+//		"\"\":  23456789012E66,\n"
+//		"\"zero\": 0,\n"
+//		"\"one\": 1,\n"
+//		"\"space\": \" \",\n"
+//		"\"quote\": \"\\\"\",\n"
+//		"\"backslash\": \"\\\\\",\n"
+//		"\"controls\": \"\\b\\f\\n\\r\\t\",\n"
+//		"\"slash\": \"/ & \\/\",\n"
+//		"\"alpha\": \"abcdefghijklmnopqrstuvwyz\",\n"
+//		"\"no value\":,\n"
+//		"\"ALPHA\": \"ABCDEFGHIJKLMNOPQRSTUVWYZ\",\n"
+//		"\"digit\": \"0123456789\",\n"
+//		"\"0123456789\": \"digit\",\n"
+//		"\"special\": \"`1~!@#$%^&*()_+-={':[,]}|;.</>?\",\n"
+//		"\"hex\": \"\\u0123\\u4567\\u89AB\\uCDEF\\uabcd\\uef4A\",\n"
+//		"\"true\": true,\n"
+//		"\"false\": false,\n"
+//		"\"null\": null,\n"
+//		"\"array\":[  ],\n"
+//		"\"object\":{  },\n"
+		"\"address\": \"50 St. James Street\",\n"
+//		"\"url\": \"http://www.JSON.org/\",\n"
+		"\"comment\": \"// /* <!-- --\",\n"     // XXX for just this, BStore.Print(1) shows the value but not the key???
+//		"\"# -- --> */\": \" \",\n"
+//		"\" s p a c e d \" :[1,2 , 3\n"
+//		"\n"
+//		",\n"
+//		"\n"
+//		"4 , 5        ,          6           ,7        ],\"compact\":[1,2,3,4,5,6,7,],\n"
+//		"\"jsontext\": \"{\\\"object with 1 member\\\":[\\\"array with 1 element\\\"]}\",\n"
+//		"\"quotes\": \"&#34; \\u0022 %22 0x22 034 &#x22;\",\n"
+//		"\"\\/\\\\\\\"\\uCAFE\\uBABE\\uAB98\\uFCDE\\ubcda\\uef4A\\b\\f\\n\\r\\t`1~!@#$%^&*()_+-=[]{}|;:',./<>?\"\n"
+//		": \"A key can be any string\"\n"
+		"}";
+	
+	BStore store{};
+	JSONP parser{};
+//	parser.SetVerbose(true);
+	bool ok = parser.Parse(testjson, store);
+	std::cout<<"parser ok? "<<ok<<std::endl;
+	std::cout<<"Store contents:\n";
+	std::cout<<"================"<<std::endl;
+	if(argc>1 && std::string(argv[1])=="1") store.Print(true);
+	else store.Print(false);
+	std::cout<<"================"<<std::endl;
+	
+	bool boolean;
+	int64_t a_numeric;
+	double d_numeric;
+	std::string a_str;
+	std::vector<int64_t> v;
+	BStore tmpstore;
+//	ok=store.Get("true",boolean);
+//	if(ok){
+//		std::cout<<"true => "<<boolean<<std::endl;
+//	}
+//	ok=store.Get("false",boolean);
+//	if(ok){
+//		std::cout<<"false => "<<boolean<<std::endl;
+//	}
+//	ok=store.Get("null",a_str);
+//	if(ok){
+//		std::cout<<"null => '"<<a_str<<"'"<<std::endl;
+//	}
+//	store.Get("integer", a_numeric);
+//	std::cout<<"integer => "<<a_numeric<<std::endl;
+//	store.Get("real", d_numeric);
+//	std::cout<<"real => "<<d_numeric<<std::endl;
+//	store.Get("zero", a_numeric);
+//	std::cout<<"zero => "<<a_numeric<<std::endl;
+//	store.Get("one", a_numeric);
+//	std::cout<<"one => "<<a_numeric<<std::endl;
+//	store.Get("e", d_numeric);
+//	std::cout<<"e => "<<d_numeric<<std::endl;
+//	store.Get("E", d_numeric);
+//	std::cout<<"E => "<<d_numeric<<std::endl;
+//	store.Get("", d_numeric);
+//	std::cout<<"'' => "<<d_numeric<<std::endl;
+//	store.Get("space", a_str);
+//	std::cout<<"space => '"<<a_str<<"'"<<std::endl;
+//	store.Get("no value", a_str);
+//	std::cout<<"no value => '"<<a_str<<"'"<<std::endl;
+//	store.Get("quotes", a_str);
+//	std::cout<<"quotes => '"<<a_str<<"'"<<std::endl;
+
+	store.Get("comment", a_str);
+	std::cout<<"comment => '"<<a_str<<"'"<<std::endl;
+	store.Get("# -- --> */", a_str);
+	std::cout<<"# -- --> */ => '"<<a_str<<"'"<<std::endl;
+
+//	ok=store.Get(" s p a c e d ", v);
+//	if(ok){
+//		std::cout<<" s p a c e d  => [";
+//		for(int i=0; i<v.size(); ++i){
+//			a_numeric=v[i];
+//			if(ok) std::cout<<a_numeric<<", ";
+//		}
+//		if(v.size()) std::cout<<"\b\b";
+//		std::cout<<"]"<<std::endl;
+//	}
+//	ok=store.Get("compact", v);
+//	if(ok){
+//		std::cout<<"compact => [";
+//		for(int i=0; i<v.size(); ++i){
+//			a_numeric=v[i];
+//			if(ok) std::cout<<a_numeric<<", ";
+//		}
+//		if(v.size()) std::cout<<"\b\b";
+//		std::cout<<"]"<<std::endl;
+//	}
+//	ok=store.Get("array", v);
+//	if(ok){
+//		std::cout<<"array => [";
+//		for(int i=0; i<v.size(); ++i){
+//			a_numeric=v[i];
+//			if(ok) std::cout<<a_numeric<<", ";
+//		}
+//		if(v.size()) std::cout<<"\b\b";
+//		std::cout<<"]"<<std::endl;
+//	}
+//	ok=store.Get("\\/\\\\\\\"\\uCAFE\\uBABE\\uAB98\\uFCDE\\ubcda\\uef4A\\b\\f\\n\\r\\t`1~!@#$%^&*()_+-=[]{}|;:',./<>?",a_str);
+//	if(ok){
+//		std::cout<<"\"\\/\\\\\\\"\\uCAFE\\uBABE\\uAB98\\uFCDE\\ubcda\\uef4A\\b\\f\\n\\r\\t`1~!@#$%^&*()_+-=[]{}|;:',./<>?\" => "
+//		         <<a_str<<std::endl;
+//	} else {
+//		std::cerr<<" couldn't find complex key "<<std::endl;
+//	}
+//	ok=store.Get("object",tmpstore);
+//	if(ok){
+//		std::cout<<"object => ";  // its empty so this prints nothing
+//		tmpstore.Print();
+//	}
+	
+	/*
+	std::cout<<"store.Has(\"SlackBot\") = "<<store.Has("SlackBot")<<std::endl;
+	if(store.Has("SlackBot")){
+		int vmenum;
+		store.Get("SlackBot",vmenum);
+		std::cout<<"SlackBot is "<<vmenum<<std::endl;
+	}
+	*/
+	
+	return 0;
+}

From 27475a69d1ba9498c4b7d60c4a9a47e9a8b77aea Mon Sep 17 00:00:00 2001
From: Marcus O'Flaherty <moflaherty1@sheffield.ac.uk>
Date: Tue, 2 Dec 2025 17:44:52 +0000
Subject: [PATCH 7/7] bugfix not recognising keys with preceding spaces

---
 JsonParser.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/JsonParser.cpp b/JsonParser.cpp
index 53b3f7a..5c18793 100644
--- a/JsonParser.cpp
+++ b/JsonParser.cpp
@@ -975,6 +975,7 @@ bool JSONP::ScanJsonObject(std::string thejson, BStore& outstore){
 		if(verbose) std::cout<<"updating iterators"<<std::endl;
 		if(next_end==thejson.length()) break;
 		next_start=next_end+1;
+		while(next_start<thejson.length() && std::isspace(thejson[next_start])) ++next_start;
 	}
 	if(verbose) std::cout<<"parsing object done"<<std::endl;