feat: add "On SortVerbose;" for extra sort info

Add information to the final sort summaries (per thread and master)
including the number of comparisons made, the number of times
the small and large buffers were sorted due to their capacities, and
the total size of the unsorted, uncompressed generated terms.

Author: jodavies

sources/compcomm.c

@@ -143,6 +143,7 @@ static KEYWORDV onoffoptions[] = {
 	,{"humanstats",	&(AC.HumanStatsFlag), 1, 0}
 	,{"humanstatistics",	&(AC.HumanStatsFlag), 1, 0}
 	,{"grccverbose", &(AC.GrccVerbose), 1, 0}
+	,{"sortverbose", &(AC.SortVerbose), 1, 0}
 };
 
 static WORD one = 1;

sources/sort.c

@@ -72,14 +72,6 @@ extern LONG nummallocs;
 extern LONG numfrees;
 #endif
 
-//#define COUNTCOMPARES
-#ifdef COUNTCOMPARES
-	// This needs to be large enough for the number of threads.
-	// It is hardcoded here, but 1024 should be enough.
-	// Enabling this has a performance impact.
-	LONG numcompares[1024];
-#endif
-
 /*
   	#] Includes : 
 	#[ SortUtilities :
@@ -164,10 +156,14 @@ void WriteStats(POSITION *plspace, WORD par, WORD checkLogType)
 		char humanGenTermsText[HUMANSTRLEN] = "";
 		char humanTermsLeftText[HUMANSTRLEN] = "";
 		char humanBytesText[HUMANSTRLEN] = "";
+		char humanUnsortedBytesText[HUMANSTRLEN] = "";
+		char humanComparisonsText[HUMANSTRLEN] = "";
 		if ( AC.HumanStatsFlag ) {
 			HumanString(humanGenTermsText, (float)(S->GenTerms), humanTermsSuffix);
 			HumanString(humanTermsLeftText, (float)(S->TermsLeft), humanTermsSuffix);
 			HumanString(humanBytesText, (float)(BASEPOSITION(*plspace)), humanBytesSuffix);
+			HumanString(humanUnsortedBytesText, (float)(S->verbUnsortedSize), humanBytesSuffix);
+			HumanString(humanComparisonsText, (float)(S->verbComparisons), humanTermsSuffix);
 		}
 
 		MLOCK(ErrorMessageLock);
@@ -315,6 +311,23 @@ void WriteStats(POSITION *plspace, WORD par, WORD checkLogType)
 			MesPrint("%s", buf);
 		}
 
+		if ( par == STATSPOSTSORT ) {
+			if ( AC.SortVerbose ) {
+				snprintf(buf, sizeof(buf), "%24s Unsorted bytes  =%11ld%s",
+					"",S->verbUnsortedSize,humanUnsortedBytesText);
+				MesPrint("%s", buf);
+				snprintf(buf, sizeof(buf), "%24s Small Buffer    =%5ld,%5ld",
+					"",S->verbSBsortTerms,S->verbSBsortCap);
+				MesPrint("%s", buf);
+				snprintf(buf, sizeof(buf), "%24s Large Buffer    =%5ld,%5ld",
+					"",S->verbLBsortPatches,S->verbLBsortCap);
+				MesPrint("%s", buf);
+				snprintf(buf, sizeof(buf), "%24s Comparisons     =%11ld%s",
+					"",S->verbComparisons,humanComparisonsText);
+				MesPrint("%s", buf);
+			}
+		}
+
 #ifdef WITHSTATS
 		MesPrint("Total number of writes: %l, reads: %l, seeks, %l"
 			,numwrites,numreads,numseeks);
@@ -362,14 +375,6 @@ int NewSort(PHEAD0)
 	}
 	if ( AR.sLevel == 0 ) {
 
-#ifdef COUNTCOMPARES
-#ifdef WITHPTHREADS
-		numcompares[AT.identity] = 0;
-#else
-		numcompares[0] = 0;
-#endif
-#endif
-
 		AN.FunSorts[0] = AT.S0;
 		if ( AR.PolyFun == 0 ) { AT.S0->PolyFlag = 0; }
 		else if ( AR.PolyFunType == 1 ) { AT.S0->PolyFlag = 1; }
@@ -408,6 +413,13 @@ int NewSort(PHEAD0)
 
 	PUTZERO(AN.OldPosOut);
 */
+	// Zero the SortVerbose counters:
+	S->verbComparisons = 0;
+	S->verbSBsortTerms = 0;
+	S->verbSBsortCap = 0;
+	S->verbLBsortPatches = 0;
+	S->verbLBsortCap = 0;
+	S->verbUnsortedSize = 0;
 	return(0);
 }
 
@@ -621,6 +633,10 @@ LONG EndSort(PHEAD WORD *buffer, int par)
 /*
 			The large buffer is too full. Merge and write it
 */
+			// Update SortVerbose counters
+			if ( S->lPatch >= S->MaxPatches ) S->verbLBsortPatches++;
+			else S->verbLBsortCap++;
+
 #ifdef GZIPDEBUG
 			MLOCK(ErrorMessageLock);
 			MesPrint("%w EndSort: lPatch = %d, MaxPatches = %d,lFill = %x, sSpace = %ld, MaxTer = %d, lTop = %x"
@@ -977,18 +993,6 @@ LONG EndSort(PHEAD WORD *buffer, int par)
 		}
 	}
 
-#ifdef COUNTCOMPARES
-	if ( AR.sLevel < 0 ) {
-#ifdef WITHPTHREADS
-		MLOCK(ErrorMessageLock);
-		MesPrint(">>>number of calls to Compare: %l (tid %d)", numcompares[AT.identity], AT.identity);
-		MUNLOCK(ErrorMessageLock);
-#else
-		MesPrint(">>>number of calls to Compare: %l", numcompares[0]);
-#endif
-	}
-#endif
-
 	return(retval);
 WorkSpaceError:
 	MLOCK(ErrorMessageLock);
@@ -2344,15 +2348,8 @@ WORD Compare1(PHEAD WORD *term1, WORD *term2, WORD level)
 	WORD prevorder;
 	WORD count = -1, localPoly, polyhit = -1;
 
-#ifdef COUNTCOMPARES
-	if ( AR.sLevel == 0 ) {
-#ifdef WITHPTHREADS
-		numcompares[AT.identity]++;
-#else
-		numcompares[0]++;
-#endif
-	}
-#endif
+	// Update SortVerbose counter
+	S->verbComparisons++;
 
 	if ( S->PolyFlag ) {
 /*
@@ -4252,6 +4249,9 @@ int StoreTerm(PHEAD WORD *term)
 	POSITION pp;
 	LONG lSpace, sSpace, RetCode, over, tover;
 
+	// Update SortVerbose counters
+	S->verbUnsortedSize += *term * sizeof(*term);
+
 	if ( ( ( AP.PreDebug & DUMPTOSORT ) == DUMPTOSORT ) && AR.sLevel == 0 ) {
 #ifdef WITHPTHREADS
 		snprintf((char *)(THRbuf),100,"StoreTerm(%d)",AT.identity);
@@ -4266,6 +4266,10 @@ int StoreTerm(PHEAD WORD *term)
 /*
 	The small buffer is full. It has to be sorted and written.
 */
+		// Update SortVerbose counters
+		if ( S->sTerms >= S->TermsInSmall ) S->verbSBsortTerms++;
+		else S->verbSBsortCap++;
+
 		tover = over = S->sTerms;
 		ss = S->sPointer;
 		ss[over] = 0;
@@ -4298,6 +4302,10 @@ int StoreTerm(PHEAD WORD *term)
 /*
 			The large buffer is too full. Merge and write it
 */
+			// Update SortVerbose counters
+			if ( S->lPatch >= S->MaxPatches ) S->verbLBsortPatches++;
+			else S->verbLBsortCap++;
+
 			if ( MergePatches(1) ) goto StoreCall;
 /*
 			pp = S->SizeInFile[1];

sources/structs.h

@@ -1156,6 +1156,12 @@ typedef struct sOrT {
     LONG SpaceLeft;             /* Space needed for still existing terms */
     LONG putinsize;             /* Size of buffer in putin */
     LONG ninterms;              /* Which input term ? */
+    LONG verbComparisons;       /* Counters for "On SortVerbose;" statistics */
+    LONG verbSBsortTerms;
+    LONG verbSBsortCap;
+    LONG verbLBsortPatches;
+    LONG verbLBsortCap;
+    LONG verbUnsortedSize;
     int MaxPatches;             /* Maximum number of patches in large buffer */
     int MaxFpatches;            /* Maximum number of patches in one filesort */
     int type;                   /* Main, function or sub(routine) */
@@ -1826,6 +1832,7 @@ struct C_const {
     int    FlintPolyFlag;          /* Use Flint for polynomial arithmetic */
     int     HumanStatsFlag;        /* Print human-readable stats in the stats print? */
     int     GrccVerbose;           /* Enable extra print statements in grcc? */
+    int     SortVerbose;           /* Enable extra sort stats information? */
 	int     doloopstacksize;
 	int     dolooplevel;
     int     CheckpointFlag;        /**< Tells preprocessor whether checkpoint code must executed.

sources/threads.c

@@ -1987,6 +1987,8 @@ void *RunSortBot(void *dummy)
 				AT.SB.FillBlock = 1;
 				AT.SB.MasterFill[1] = AT.SB.MasterStart[1];
 				SETBASEPOSITION(AN.theposition,0);
+				// Reset the sortbot comparison count
+				AT.SS->verbComparisons = 0;
 				break;
 /*
 			#] INISORTBOT : 
@@ -4052,10 +4054,23 @@ int MasterMerge(void)
 	fin->handle = -1;
 	position = S->SizeInFile[0];
 	MULPOS(position,sizeof(WORD));
+
+	// Collect global sort statistics information from the threads.
+	// The total GenTerms is the sum of the thread GenTerms.
+	// The total small/large buffer sort info is the sum of the thread info.
+	// The total comparison count is the sum of the thread counts.
+	// The total unsorted size is the sum of the total generated terms sizes
 	S->GenTerms = 0;
 	for ( j = 1; j <= numberofworkers; j++ ) {
 		S->GenTerms += AB[j]->T.SS->GenTerms;
+		S->verbComparisons += AB[j]->T.SS->verbComparisons;
+		S->verbSBsortTerms += AB[j]->T.SS->verbSBsortTerms;
+		S->verbSBsortCap += AB[j]->T.SS->verbSBsortCap;
+		S->verbLBsortPatches += AB[j]->T.SS->verbLBsortPatches;
+		S->verbLBsortCap += AB[j]->T.SS->verbLBsortCap;
+		S->verbUnsortedSize += AB[j]->T.SS->verbUnsortedSize;
 	}
+
 	WriteStats(&position,STATSPOSTSORT,NOCHECKLOGTYPE);
 	Expressions[AR0.CurExpr].counter = S->TermsLeft;
 	Expressions[AR0.CurExpr].size = position;
@@ -4193,10 +4208,26 @@ int SortBotMasterMerge(void)
 	}
 	position = S->SizeInFile[0];
 	MULPOS(position,sizeof(WORD));
+
+	// Collect global sort statistics information from the threads.
+	// The total GenTerms is the sum of the thread GenTerms.
+	// The total small/large buffer sort info is the sum of the thread info.
+	// The total comparison count is the sum of the thread and sortbot counts.
+	// The total unsorted size is the sum of the total generated terms sizes
 	S->GenTerms = 0;
 	for ( j = 1; j <= numberofworkers; j++ ) {
 		S->GenTerms += AB[j]->T.SS->GenTerms;
+		S->verbComparisons += AB[j]->T.SS->verbComparisons;
+		S->verbSBsortTerms += AB[j]->T.SS->verbSBsortTerms;
+		S->verbSBsortCap += AB[j]->T.SS->verbSBsortCap;
+		S->verbLBsortPatches += AB[j]->T.SS->verbLBsortPatches;
+		S->verbLBsortCap += AB[j]->T.SS->verbLBsortCap;
+		S->verbUnsortedSize += AB[j]->T.SS->verbUnsortedSize;
 	}
+	for ( j = numberofworkers+1; j <= numberofworkers+numberofsortbots; j++ ) {
+		S->verbComparisons += AB[j]->T.SS->verbComparisons;
+	}
+
 	S->TermsLeft = numberofterms;
 	WriteStats(&position,STATSPOSTSORT,NOCHECKLOGTYPE);
 	Expressions[AR.CurExpr].counter = S->TermsLeft;