Updates to THcTimeSyncEvtHandler

  THcTimeSyncEvtHandler is a toolbox to study and correct various
  synchronization issues.
  It checks that the timestamp in the FADC250s agrees with the timestamp
  in the TI modules.
  It also checks if the event counter in the 1190 modules agrees with the
  event counter in the TI modules.
  It can be used to correct the 1190 synchronization problem we have
  seen in ROC3 by writing out a corrected data file.  See the next commit
  for an example of how it is used.

src/THcTimeSyncEvtHandler.cxx

@@ -19,24 +19,42 @@
 
 \author Stephen Wood (saw@jlab.org)
 */
+/* This version will deal with bad run coin_all_03302.dat.
+
+That has an event with most of a noisy set of 1190 data in ROC 3.
+The next event has the remainder of the noisy data.  It's good 1190 data
+is in the next event.
+So the 1190 data for the first event with noisy data should be discarded.
+(Empty bank).  The next event should be cached, and then written out when
+the next event comes.
+*/
 
 #include "THcTimeSyncEvtHandler.h"
 #include "THaEvData.h"
 #include "THaGlobals.h"
 #include "THcGlobals.h"
 #include "THcParmList.h"
+#include "THaCodaFile.h"
+#include "THaRunBase.h"
 #include <cstring>
 #include <cstdio>
 #include <cstdlib>
 #include <iostream>
 #include <iomanip>
+//#include "evio.h"
 
 using namespace std;
+using namespace Decoder;
 
 THcTimeSyncEvtHandler::THcTimeSyncEvtHandler(const char *name, const char* description)
   : THaEvtTypeHandler(name,description)
 {
   ExpectedOffsetMap.clear();
+  fBadROC = -1;
+  fResync = kTRUE;
+  fBadSyncSizeTrigger = 450;
+  fCodaOut = 0;
+  fLastEventWasSync = kFALSE;
 }
 
 THcTimeSyncEvtHandler::~THcTimeSyncEvtHandler()
@@ -55,16 +73,29 @@ THcTimeSyncEvtHandler::~THcTimeSyncEvtHandler()
 Int_t THcTimeSyncEvtHandler::Analyze(THaEvData *evdata)
 {
 
-  if ( !IsMyEvent(evdata->GetEvType()) ) return -1;
+  //  cout << evdata->GetEvType() << " " << evdata->GetEvLength() << endl;
+
+  // If filtering data file, pass through all events we are not dealing with
+  if ( !IsMyEvent(evdata->GetEvType()) ) {
+    if(fCodaOut) {
+      fCodaOut->codaWrite(evdata->GetRawDataBuffer());
+    }
+    return -1;
+  }
+
 
   if (fDebug) cout << "------------------\n  Time Syncronization Checker"<<endl;
 
   Int_t evlen = evdata->GetEvLength();
+
   UInt_t *rdata = (UInt_t*) evdata->GetRawDataBuffer();
 
+  UInt_t *pslippedbank=0;
+
   UInt_t *p = (UInt_t*) rdata;
   UInt_t *plast = p+*p;		// Index to last word in the bank
   Int_t roc = -1;
+  Bool_t issyncevent=kFALSE;
 
   CrateTimeMap.clear();
 
@@ -87,6 +118,7 @@ Int_t THcTimeSyncEvtHandler::Analyze(THaEvData *evdata)
       // a bank within a ROC containing data from modules of a single type
       // Look for TI and FADC banks.
       UInt_t tag = (*p>>16) & 0xffff;
+      UInt_t num = *p & 0xff;
       UInt_t *pnext = p+banklen;	// Next bank
       p++;			// First data word
       if(tag==4) { // This is a TI blob banks
@@ -96,13 +128,18 @@ Int_t THcTimeSyncEvtHandler::Analyze(THaEvData *evdata)
         if(ifill) {
           p++; banklen--;  // Skip filler word
         }
+	CrateTimeMap[roc]->ti_evcount = p[3];
         if(banklen>=5) {   // Need bank header, at least 2 TI  headers, the  trailer and 2 data words
           UInt_t titime = p[4];
 	  if(fDebug) cout << roc << ": TItime " << titime << endl;
           CrateTimeMap[roc]->has_ti_ttime = kTRUE;
           CrateTimeMap[roc]->ti_ttime = titime;
         }
-      } else if (tag==250) { // This is an FADC banks
+      } else if (tag==3801) {
+	if(fResync && num==1) {
+	  issyncevent = kTRUE;
+	}
+      } else if (tag==250) { // This is an FADC bank
 	if(fDebug) cout << roc << ": FADC" << endl;
         // Walk through this bank looking for FADC headers.
         Int_t slot=-1;
@@ -117,7 +154,7 @@ Int_t THcTimeSyncEvtHandler::Analyze(THaEvData *evdata)
 	    {
 	      UInt_t fadctime = ((*p)&0xFFFFFF) + (((*(p+1))&0xFF)<<24);
 	      if(fDebug) cout << "    " << slot << ": " << fadctime << endl;
-        CrateTimeMap[roc]->fadcTimesMap[slot] = fadctime;
+	      CrateTimeMap[roc]->fadcTimesMap[slot] = fadctime;
 	      p += 2;
 	      break;
 	    }
@@ -126,6 +163,28 @@ Int_t THcTimeSyncEvtHandler::Analyze(THaEvData *evdata)
 	    break;
 	  }
 	}
+      } else if (tag==1190) {	// Bank with CAEN 1190 TDCs
+	if(roc==fBadROC) {		// Point to slipped bank
+	  if(fSlippage) {
+	    pslippedbank = p-2;
+	    //	    cout << banklen << " " << pslippedbank[0] << endl;
+	  } else {
+	    if(banklen > fBadSyncSizeTrigger) {
+	      cout << "Slippage enabled at event " << evdata->GetEvNum() << endl;
+	      fSlippage = 1;
+	    }
+	  }
+	}
+	if(fDebug) cout << roc << ": 1190" << endl;
+	// Walk through this bank looking for TDC headers
+	while(p<pnext) {
+	  if((*p & 0xf8000000) == 0x40000000) {
+	    Int_t slot= *p & 0x1f;
+	    Int_t evcount = (*p >> 5) & 0x3fffff;
+	    CrateTimeMap[roc]->ftdcEvCountMap[slot] = evcount;
+	  }
+	  p++;
+	}
       }
       p=pnext;    // Skip to next bank
     } else {
@@ -135,9 +194,140 @@ Int_t THcTimeSyncEvtHandler::Analyze(THaEvData *evdata)
 
   if(fFirstTime) {
     InitStats();
+    fLastEvent[0] = 0;
     fFirstTime = kFALSE;
+    fDumpNew=2;
+  }
+  if(issyncevent) cout << "SYNC event" << endl;
+  AccumulateStats(fLastEventWasSync);
+  fLastEventWasSync = issyncevent;
+
+  if(!fCodaOut) return(1);
+  
+  if(fSlippage > 0) {
+    if(fLastEvent[0]>0) { // Now slipping.  Output corrected event
+      fWriteDelayed=kTRUE;
+      //      cout << "Will write corrected event" << endl;
+    } else {
+      cout << "Skipping event " << evdata->GetEvNum() << endl;
+    }
+  } else {			// Not slipping yet, just copy event
+    if(fCodaOut) {
+      fCodaOut->codaWrite(evdata->GetRawDataBuffer());
+    }
+  }    
+
+  if(fWriteDelayed) {
+
+    // Dump event before fixing
+    //    if(fDumpNew > 0) {
+    //      cout << "Unmodified event" << endl;
+    //      for(UInt_t i=0;i<=fLastEvent[0];i++) {
+    //	if(i%5 == 0) cout<<endl<<dec<<i<<": ";
+    //	cout << hex << setw(10) << fLastEvent[i];
+    //      }
+    //    }
+    //    cout << dec << endl;
+
+    // Use fLastEvent with current pslipped bank to write out fixed event
+    // Find the 1190 bank.  Shift everything beyond that bank forward or
+    // or back to make room for the 1190 bank from the current event.
+    // Copy the current 1190 bank into fLastEvent.  Write the event.
+    p = fLastEvent;
+    plast = p+*p;
+    roc = -1;
+    UInt_t *poverwrite=0;
+    Int_t replacementlen;
+    Int_t banklen;
+    UInt_t *rocbanklenp=0;
+    UInt_t *roc3banklenp=0;
+
+    //    cout << "Old Size: " << fLastEvent[0] << " " << pslippedbank[0] << " ";
+    while(p<plast) {
+      banklen = *p;
+      p++;
+      if((*p & 0xff00) == 0x1000) {	// Bank Containing banks
+	if((fLastEvent[0]+1)-*(p-1) > 1) { // Don't use overall event header
+	  roc = (*p>>16) & 0xf;
+	  rocbanklenp = p-1;	// Pointer to rocbank length
+	}
+      p++;				// Now pointing to a bank in the bank
+      } else if (((*p & 0xff00) == 0x100) && (*p != 0xC0000100)) {
+	// Bank containing integers.
+	// This is either ROC bank containing integers or
+	// a bank within a ROC containing data from modules of a single type
+	// Look for TI and FADC banks.
+	UInt_t tag = (*p>>16) & 0xffff;
+	UInt_t *pnext = p+banklen;	// Next bank
+	p++;			// First data word
+	if(tag==1190 && roc==fBadROC) { // The TDC bank we want to replace
+	  replacementlen=pslippedbank[0];
+	  roc3banklenp=rocbanklenp; // Save pointer to ROC bank header
+	  poverwrite=p-2;	// Where to write the slipped bank
+	  break;
+	}
+	p=pnext;
+      } else {
+	p=p+*(p-1);
+      }
+    }
+    Int_t icopied=0;
+    if(poverwrite) {
+      //      cout << banklen << endl;
+      if(replacementlen < banklen) { // Shift data after bank down
+	p = poverwrite+banklen+1;    // Data to be shifted
+	//	cout << "Shifting " << (p-fLastEvent) << " to " << (p+replacementlen-banklen-fLastEvent) << endl;
+	while(p<=plast) {
+	  //	  cout << (p+replacementlen-banklen-fLastEvent) << ": " << hex << *p << dec << endl;
+	  *(p + replacementlen - banklen) = *p;
+	  p++;
+	  icopied++;
+	}
+      } else if (replacementlen > banklen) { // Shift data up
+	p = plast;
+	//	UInt_t *pfirst = poverwrite + replacementlen + 1;
+	UInt_t *pfirst = poverwrite + banklen + 1;
+	//	cout << "Shifting " << (plast-pfirst+1) << " words " << (pfirst-fLastEvent) << " to " << (pfirst+replacementlen-banklen-fLastEvent) << endl;
+	while(p>=pfirst) {
+	  //	  cout << (p+replacementlen-banklen-fLastEvent) << ": " << hex << *p << dec << endl;
+	  *(p+replacementlen-banklen) = *p;
+	  p--;
+	  icopied++;
+	}
+      }
+      fLastEvent[0] += replacementlen-banklen; // Correct overall event length
+      *roc3banklenp += replacementlen-banklen; // Also ROC3 bank length
+      //      cout << "Copying " << replacementlen+1 << " words of new bank to " << poverwrite-fLastEvent << endl;
+      for(Int_t i=0;i<=replacementlen;i++) { // Copy slipped bank into last event
+	poverwrite[i] = pslippedbank[i];
+      }
+    }
+    //    cout << "New Size: " << fLastEvent[0] << "   copied " << icopied << endl;
+    if(fCodaOut) {
+      //      if(fDumpNew > 0) {
+      //	for(UInt_t i=0;i<=fLastEvent[0];i++) {
+      //	  if(i%5 == 0) cout<<endl<<dec<<i<<": ";
+      //	  cout << hex << setw(10) << fLastEvent[i];
+      //	}
+      //	fDumpNew--;
+      //      }
+      //      cout << dec << endl;
+      fCodaOut->codaWrite(fLastEvent);
+      if(issyncevent) {		// If this was a sync event, write it out and stop rewriting
+	cout << "Run back in sync at event " << evdata->GetEvNum() << endl;
+	fCodaOut->codaWrite(evdata->GetRawDataBuffer());
+	fSlippage = 0;
+	fLastEvent[0] = 0;
+	fWriteDelayed = kFALSE;
+      }
+    }
+  }
+  if(fSlippage>0) {		// Just handle slippage of 1 now
+    for(UInt_t i=0;i<=rdata[0];i++) {
+      fLastEvent[i]=rdata[i];
+    }
+    //    cout << "Cached event " << evdata->GetEvNum() << " " << fLastEvent[0] << endl;
   }
-  AccumulateStats();
 
   return 1;
 }
@@ -186,10 +376,19 @@ void THcTimeSyncEvtHandler::InitStats() {
         CrateStatsMap[roc]->fadcLateSlipCountMap[slot] = 0;
       }
     }
+    if(roctimes->ftdcEvCountMap.size()>0) {
+      if(fDebug) cout << endl << " 1190";
+      std::map<Int_t, UInt_t>::iterator itt = roctimes->ftdcEvCountMap.begin();
+      while(itt != roctimes->ftdcEvCountMap.end()) {
+        Int_t slot = itt->first;
+	itt++;
+	CrateStatsMap[roc]->ftdcEvCountWrongMap[slot] = 0;
+      }
+    }
   }
 }
 
-void THcTimeSyncEvtHandler::AccumulateStats() {
+void THcTimeSyncEvtHandler::AccumulateStats(Bool_t sync) {
   fNEvents++;
   // Get trigger time from master CrateInfo
   UInt_t master_ttime = CrateTimeMap[fMasterRoc]->ti_ttime;
@@ -216,6 +415,28 @@ void THcTimeSyncEvtHandler::AccumulateStats() {
         }
       }
     }
+    if(rocstats->ftdcEvCountWrongMap.size()>0) {
+      std::map<Int_t, Int_t>::iterator itt = rocstats->ftdcEvCountWrongMap.begin();
+      while(itt != rocstats->ftdcEvCountWrongMap.end()) {
+	Int_t slot = itt->first;
+	itt++;
+	if(rocstats->ftdcEvCountOffsetMap.find(slot)==rocstats->ftdcEvCountOffsetMap.end()) {
+	  rocstats->ftdcEvCountOffsetMap[slot] = 1;
+	}
+	Int_t cdiff = (CrateTimeMap[roc]->ti_evcount & 0x3fffff) -
+	  ((CrateTimeMap[roc]->ftdcEvCountMap[slot]+rocstats->ftdcEvCountOffsetMap[slot])&0x3fffff);
+	if(sync) { // Need to do this check on the event after the sync event too
+	  if(cdiff>2) {
+	    cout << "ROC/Slot " << roc << "/" << slot << " count diff correction " << cdiff << endl;
+	    rocstats->ftdcEvCountOffsetMap[slot] += cdiff;
+	    cdiff = 0;
+	  }
+	}
+	if(cdiff != 0) {
+	  rocstats->ftdcEvCountWrongMap[slot]++;
+	}
+      }
+    }
   }
 }
 void THcTimeSyncEvtHandler::PrintStats()
@@ -244,6 +465,16 @@ void THcTimeSyncEvtHandler::PrintStats()
 	cout << "    " << slot << " " << rocstats->fadcOffsetMap[slot] << "    " << earlyslips << "    " << lateslips << endl;
       }
     }
+    std::map<Int_t, Int_t>::iterator itw = rocstats->ftdcEvCountWrongMap.begin();
+    while(itw != rocstats->ftdcEvCountWrongMap.end()) {
+      Int_t slot = itw->first;
+      Int_t wrongcount = itw->second;
+      itw++;
+      if(wrongcount > 0) {
+	cout << "    " << slot << " " << wrongcount << endl;
+      }
+    }
+	
   }
   cout << "-------------------------------------------------------------------" << endl;
 }
@@ -261,17 +492,58 @@ THaAnalysisObject::EStatus THcTimeSyncEvtHandler::Init(const TDatime& date)
   cout << "Howdy !  We are initializing THcTimeSyncEvtHandler !!   name =   "<<fName<<endl;
 
   if(eventtypes.size()==0) {
-    eventtypes.push_back(125);  // what events to look for
+    eventtypes.push_back(1);  // If no event types specified, 
+    eventtypes.push_back(2);  // check timestamp synchronization on these event types
+    eventtypes.push_back(3);
+    eventtypes.push_back(4);
+    eventtypes.push_back(5);
+    eventtypes.push_back(6);
+    eventtypes.push_back(7);
   }
 
   fFirstTime = kTRUE;
   fMasterRoc = -1;
   fNEvents = 0;
   CrateTimeMap.clear();
+  fSlippage=0;
+  fWriteDelayed=kFALSE;
+  fLastEvent[0]=0;
   //CrateStatsMap.clear();
+  if(fCodaOut && fBadROC < 0) {
+    Warning(Here("Init"), "Sync filtering requested, but bad ROC not specified");
+  }
 
   fStatus = kOK;
   return kOK;
 }
 
+Int_t THcTimeSyncEvtHandler::End( THaRunBase* r) {
+  if(!fCodaOut) {
+    return 0;
+  }
+  return fCodaOut->codaClose();
+}
+
+Int_t THcTimeSyncEvtHandler::SetRewriteFile(const char *filename) {
+  if(filename==0 || strlen(filename)==0) {
+    fCodaOut = 0;
+    cout << "THcTimeSyncEvtHandler sync filtering disabled" << endl;
+  } else {
+    TString ts=filename;
+    fCodaOut = new THaCodaFile;
+    if (!fCodaOut) {
+      Error(Here("SetRewriteFile"),"Cannot create CODA output file object. "
+	    "Something is very wrong." );
+      return -2;
+    }
+    if ( fCodaOut->codaOpen(ts, "w", 1) ) {
+      Error(Here("SetRewriteFile"),"Cannot open CODA file %s for writing.",ts.Data());
+      return -3;
+    }
+    cout << "THcTimeSYncEvtHandler writing sync filtered coda file to " << ts <<endl;
+  }
+  return(0);
+}
+
+
 ClassImp(THcTimeSyncEvtHandler)

src/THcTimeSyncEvtHandler.h

@@ -8,10 +8,14 @@
 /////////////////////////////////////////////////////////////////////
 
 #include "THaEvtTypeHandler.h"
+#include "Decoder.h"
 #include <string>
 #include <vector>
 #include <map>
 
+//class THaRunBase;
+//class THaEvData;
+
 class THcTimeSyncEvtHandler : public THaEvtTypeHandler {
 
 public:
@@ -24,23 +28,43 @@ public:
   virtual void PrintStats();
   virtual void SetExpectedOffset(Int_t roc, Int_t offset);
   virtual void AddExpectedOffset(Int_t roc, Int_t offset);
-
- //  Float_t GetData(const std::string& tag);
+  virtual Int_t End( THaRunBase* r=0 );
+  virtual Int_t SetRewriteFile(const char *filename);
+  virtual void SetBadROC(Int_t roc) {fBadROC = roc;}
+  virtual void SetResync(Bool_t b) {fResync = b;}
+  virtual void SetBadSyncSizeTrigger(Int_t sizetrigger) {fBadSyncSizeTrigger = sizetrigger;}
 
 private:
 
   virtual void InitStats();
-  virtual void AccumulateStats();
+  virtual void AccumulateStats(Bool_t sync);
 
   Bool_t fFirstTime;
   Int_t fMasterRoc; // ROC with the TI master
   Int_t fNEvents;   // Number of events analyzed
+  Int_t fSlippage;
+  UInt_t fLastEvent[32000];
+  UInt_t fSlippingBank[32000];
+  Bool_t fWriteDelayed;
+  Int_t fDumpNew;
+  Int_t fBadROC;		// ROC to check and filter sync problems
+  Bool_t fResync;		// If true, stop correcting events on sync
+  Int_t fBadSyncSizeTrigger;
+  Bool_t fLastEventWasSync;	// True when last event was sync event
+
+  Decoder::THaCodaFile* fCodaOut; // The CODA output file
+  Int_t handle;
 
   typedef struct RocTimes {
     Bool_t has_ti_ttime;
     UInt_t ti_ttime;
+    UInt_t ti_evcount;
     std::map<Int_t, UInt_t> fadcTimesMap;
-    RocTimes() : has_ti_ttime(kFALSE) {fadcTimesMap.clear();}
+    std::map<Int_t, UInt_t> ftdcEvCountMap;
+    RocTimes() : has_ti_ttime(kFALSE) {
+      fadcTimesMap.clear();
+      ftdcEvCountMap.clear();
+    }
   } RocTimes_t;
 
   typedef struct RocStats {
@@ -51,12 +75,16 @@ private:
     std::map<Int_t, Int_t> fadcOffsetMap;
     std::map<Int_t, Int_t> fadcEarlySlipCountMap;
     std::map<Int_t, Int_t> fadcLateSlipCountMap;
+    std::map<Int_t, Int_t> ftdcEvCountWrongMap;
+    std::map<Int_t, Int_t> ftdcEvCountOffsetMap;
     RocStats() : ti_ttime_offset(0), ti_earlyslipcount(0), ti_lateslipcount(0),
       fadc_expected_offset(0)
     {
       fadcOffsetMap.clear();
       fadcEarlySlipCountMap.clear();
       fadcLateSlipCountMap.clear();
+      ftdcEvCountWrongMap.clear();
+      ftdcEvCountOffsetMap.clear();
     }
   } RocStats_t;