make statefiles and change of ncpus robust

Author: tromp

Legal.hs

@@ -1,15 +1,28 @@
+module Legal where
 import Data.Array
 import Data.List
 import Data.Char
 import Numeric
+import Data.FixedPoint
 
-alpha  = 0.850639925845714538
+abls :: [(Int,FixedPoint10241024,FixedPoint10241024,FixedPoint10241024)]
+abls = abl 1 squarish where
+  abl n (nn:nn1:abl1@(n1n1:_)) = (n,a,b,l) : abl (n+1) abl1 where
+    l = fromIntegral nn * fromIntegral n1n1 / (fromIntegral nn1)^2
+    b = fromIntegral nn1 / (fromIntegral nn * l^n)
+    a = fromIntegral nn / (b^(2*n) * l^(n^2))
+  abl _ _ = []
+
+main = do
+  mapM_ print abls
+
+alpha  = 0.8506399258457126148173084786343
 logalpha = logBase 10 alpha
 
-beta   = 0.96553505933837387
+beta   = 0.96553505933837393063017090430952
 logbeta = logBase 10 beta
 
-lambda = 2.97573419204335724938
+lambda = 2.97573419204335724938110448420706
 loglambda = logBase 10 lambda
 
 legal m n = alpha * beta**(m+n) * lambda**(m*n)
@@ -32,3 +45,6 @@ legal2 = recurrence [-1,2,20,-13,31,-16,10] [1,5,57,489,4125,35117,299681]
 legal3 = recurrence [-5,73,100,-1402,1014,-5352,-2490,6018,-4020,1766,9083,-19993,22072,-16646,9426,-3750,1171,-233,33] [1,15,489,12675,321689,8180343,208144601,5296282323,134764135265,3429075477543,87252874774409,2220150677358587,56491766630430761,1437433832683612783,36575525011037967769,930664771769562054147,23680778803620700205625,602557764897193682879119,15332091188757329557096929]
 
 ternary n = showIntAtBase 3 intToDigit n ""
+
+
+squarish = [1,5,57,489,12675,321689,24318165,1840058693,414295148741,93332304864173,62567386502084877,41945191530093646965,83677847847984287628595,166931297609667912727898521,990966953618170260281935463385,5882748866432370655674372752123193,103919148791293834318983090438798793469,1835738613899845421140262364853644706891109,96498428501909654589630887978835098088148177857,5072588588647327658457862518216696854885169490987149,793474866816582266820936671790189132321673383112185151899,124118554774307129694783556890846966815009879092863579679259393,57774258489513238998237970307483999327287210756991189655942651331169,26892554058860272116972562366415920138007095980551558908000982332405743333,37249792307686396442294904767024517674249157948208717533254799550970595875237705,51595955665685681166597566866805181435596339502695699293823422273656970477373415200373,212667732900366224249789357650440598098805861083269127196623872213228196352455447575029701325,876571894704740435776253865561651594678577903166188258472955681125289495868953613359454403019145877,10751464308361383118768413754866123809733788820327844402764601662870883601711298309339239868998337801509491,131870512244645759297888472329947870580266256924485681728458086578687538959472921550847035733890182662513180743513,4813066963822755416429056022484299646486874100967249263944719599975607459850502222039591149331431805524655467453067042377,175669398745227675074920433327787366374551886098434473836510647159450821039563378374569811240252763776406712988379278644250456677,19079388919628199204605726181850465220151058338147922243967269231944059187214767997105992341735209230667288462179090073659712583262087437,2072205427619023303039587520236390121754274072718784609433998196933282608067036314403465202963700297341152216286750576593627459392979397487964077,669723114288829212892740188841706543509937780640178732810318337696945624428547218105214326012774371397184848890970111836283470468812827907149926502347633,216450089279078275314395453480468424469694873576469893709517750563261490751122922463339745178577954008324586419548071995019779454584564790800309660950831580481393]

Makefile

@@ -4,8 +4,8 @@ GFLAGS = -Wall -g -m64
 
 all:   	start legal legalg tar NIC
 
-start:	start.c states.c modulus.h Makefile
-	cc $(CFLAGS) -o start start.c states.c
+start:	start.c outstream.c modadd.h modulus.h partition.c partition.h states.c states.h sortstate.c sortstate.h Makefile
+	cc $(CFLAGS) -o start start.c states.c sortstate.c outstream.c partition.c modadd.c
 
 instream:	instream.c instream.h modadd.h modulus.h Makefile
 	cc -DMAININSTREAM $(CFLAGS) -o instream instream.c modadd.c

counts

@@ -244,3 +244,7 @@ S3  1
 N1  2
 
 :'a,'bs/2/O /g
+
+0 0 0 0 2 2 2 0 1 1 0 0 2 0 1 1 1 0 1
+2 1 1 2 1 2 0 0 2 1 2 1
+

gocheck

@@ -39,6 +39,7 @@ while(<>) {
 my $prevnlstates = 1;
 my $prevparts = 1;
 for my $yx (sort keys %cell) {
+  last unless $cell{$yx}->{size};
   my $nlstates = $cell{$yx}->{needy}+$cell{$yx}->{legal};
   my $nstates = $cell{$yx}->{newill} + $nlstates;
   my $diff = abs(($nstates - 3*$prevnlstates) % $mod);

instream.c

@@ -3,21 +3,20 @@
 #include <stdlib.h>
 #include <stdint.h>
 #include <string.h>
+#include <strings.h>
+#include <glob.h>
 #include "instream.h"
 #include "modadd.h"
 
 #define MAXSTREAMS 1024
-#ifndef FANIN
-#define FANIN 1
-#endif
 
 struct goin {
   uint64_t modulus;
   uint64_t totalin;
   statebuf stream[MAXSTREAMS];
   statebuf *heap[MAXSTREAMS];
   int nstreams;
-  int ncpus;
+  int incpus;
   int cpuid;
 };
 
@@ -126,39 +125,59 @@ void deletemin(goin *gin)
   hpreplace(gin, 0);
 }
 
-goin *openstreams(char *inbase, int ncpus, int cpuid, uint64_t modulus) {
-
+goin *openstreams(char *inbase, int incpus, int ncpus, int cpuid, uint64_t modulus) {
   char inname[FILENAMELEN];
   statebuf *sb;
   FILE *fp;
-  int from,to,j;
+  int from,to,j,nscan;
   goin *gin;
+  glob_t stateglob;
+  uint64_t prevstate,state;
 
   gin = (goin *)calloc(1,sizeof(goin));
   if (gin == NULL) {
     printf ("Failed to allocate goin record of size %d\n", (int)sizeof(goin));
     exit(1);
   }
   gin->totalin = 0LL;
-  gin->ncpus = ncpus;
+  gin->incpus = incpus;
   gin->cpuid = cpuid;
   gin->modulus = modulus;
   sb = &gin->stream[gin->nstreams = 0];
-  for (from=0; from<FANIN*ncpus; from++) {
-    for (to=cpuid; to<FANIN*ncpus; to+=ncpus) {
-      for (j=0; ; j++) {
-        sprintf(inname,"%s/fromto.%d.%d/%d",inbase,from,to,j); 
-        if (!(fp = fopen(inname, "r"))) {
-          // printf("Failed to open %s\n",inname);
-          break;
+  int limto = incpus < ncpus ? ncpus : incpus;
+  for (from=0; from<incpus; from++) {
+    for (to=cpuid; to<limto; to+=ncpus) {
+      prevstate = 0L;
+      for (j=0; prevstate!=FINALSTATE; j++) {
+        sprintf(inname,"%s/fromto.%d.%d/%d.*",inbase,from,to,j); 
+        stateglob.gl_matchc = 2;
+        glob(inname, GLOB_LIMIT, NULL, &stateglob);
+        if (stateglob.gl_matchc != 1) {
+          printf ("%d files matching %s\n", stateglob.gl_matchc, inname);
+          exit(1);
+        }
+        if (!(fp = fopen(stateglob.gl_pathv[0], "r"))) {
+          printf("Failed to open %s\n",stateglob.gl_pathv[0]);
+          exit(1);
         }
         if (gin->nstreams == MAXSTREAMS) {
           printf ("#inputfiles exceeds MAXSTREAMS (%d)\n", MAXSTREAMS);
           exit(1);
         }
-        // printf("opened %s\n", inname);
+        nscan = sscanf(rindex(stateglob.gl_pathv[0],'.')+1, "%lo", &state); 
+        if (nscan != 1) {
+          printf ("No state suffix in %s\n", stateglob.gl_matchc);
+          exit(1);
+        }
+        if (state <= prevstate) {
+          printf ("States %lo %lo out of order.\n", prevstate, state);
+          exit(1);
+        }
+        prevstate = state;
+        // printf("opened %s state %lo\n", stateglob.gl_pathv[0],state);
         sb->fp = fp;
-        strncpy(sb->fname, inname, FILENAMELEN);
+        strncpy(sb->fname, stateglob.gl_pathv[0], FILENAMELEN);
+        globfree(&stateglob);
         hpinsert(gin, gin->nstreams++, sb++);
       }
     }
@@ -178,10 +197,10 @@ int main(int argc, char *argv[])
   char inbase[64];
   statebuf *mb;
   goin gin;
-  int ncpus, cpuid;
+  int incpus, ncpus, cpuid;
 
-  if (argc!=7) {
-    fprintf(stderr, "usage: %s width imod y x ncpus cpuid\n", argv[0]);
+  if (argc!=8) {
+    fprintf(stderr, "usage: %s width imod y x incpus ncpus cpuid\n", argv[0]);
     exit(1);
   }
   wd = atoi(argv[1]);
@@ -193,19 +212,20 @@ int main(int argc, char *argv[])
   modulus = -(uint64_t)modulusdeltas[modidx];
   y = atoi(argv[3]);
   x = atoi(argv[4]);
-  ncpus = atoi(argv[5]);
+  incpus = atoi(argv[5]);
+  ncpus = atoi(argv[6]);
   if (ncpus < 1 || ncpus > MAXCPUS) {
     fprintf (stderr, "#cpus %d not in range [0,%d]\n", ncpus, MAXCPUS);
     exit(1);
   }
-  cpuid = atoi(argv[6]);
+  cpuid = atoi(argv[7]);
   if (cpuid < 0 || cpuid >= ncpus) {
     fprintf (stderr, "cpuid %d not in range [0,%d]\n", ncpus, ncpus-1);
     exit(1);
   }
 
   sprintf(inbase,"%d.%d/yx.%02d.%02d",wd,modidx,y,x);
-  gin = openstreams(inbase, ncpus, cpuid, modulus))
+  gin = openstreams(inbase, incpus, ncpus, cpuid, modulus))
   if (!nstreams(gin))
     fprintf (stderr, "wanring: no input files\n");
   for (nin=0LL; (mb = minstream(gin))->state != FINALSTATE; nin++) {

instream.h

@@ -18,4 +18,4 @@ uint64_t nstreams(goin *gin);
 uint64_t totalread(goin *gin);
 void deletemin(goin *gin);
 statebuf *minstream(goin *gin);
-goin *openstreams(char *inbase, int ncpus, int cpuid, uint64_t modulus);
+goin *openstreams(char *inbase, int incpus, int ncpus, int cpuid, uint64_t modulus);

legal.c

@@ -15,7 +15,7 @@
 
 int main(int argc, char *argv[])
 {
-  int ncpus,cpuid,modidx,width,y,x,nextx,tsizelen;
+  int incpus,ncpus,cpuid,modidx,width,y,x,nextx,tsizelen;
   uint64_t msize,modulus,nin;
   char c,*tsizearg,inbase[64],outbase[64];
   uint64_t nnewillcnt, newstates[3]; 
@@ -27,8 +27,8 @@ int main(int argc, char *argv[])
   goout *go;
 
   assert(sizeof(uint64_t)==8);
-  if (argc!=8) {
-    printf("usage: %s width modulo_index y x ncpus cpuid maxtreesize[kKmMgG]\n",argv[0]);
+  if (argc!=9) {
+    printf("usage: %s width modulo_index y x incpus ncpus cpuid maxtreesize[kKmMgG]\n",argv[0]);
     exit(1);
   }
   setwidth(width = atoi(argv[1]));
@@ -41,17 +41,18 @@ int main(int argc, char *argv[])
   y = atoi(argv[3]);
   x = atoi(argv[4]);
   nextx = (x+1) % width;
-  ncpus = atoi(argv[5]);
+  incpus = atoi(argv[5]);
+  ncpus = atoi(argv[6]);
   if (ncpus < 1 || ncpus > MAXCPUS) {
     printf ("#cpus %d not in range [0,%d]\n", ncpus, MAXCPUS);
     exit(1);
   }
-  cpuid = atoi(argv[6]);
+  cpuid = atoi(argv[7]);
   if (cpuid < 0 || cpuid >= ncpus) {
     printf("cpuid %d not in range [0,%d]\n", ncpus, ncpus-1);
     exit(1);
   }
-  tsizelen = strlen(tsizearg = argv[7]);
+  tsizelen = strlen(tsizearg = argv[8]);
   if (!isdigit(c = tsizearg[tsizelen-1]))
     tsizearg[tsizelen-1] = '\0';
    msize = atol(tsizearg);
@@ -67,35 +68,32 @@ int main(int argc, char *argv[])
   }
 
   sprintf(inbase,"%d.%d/yx.%02d.%02d",width,modidx,y,x); 
-  gin = openstreams(inbase, ncpus, cpuid, modulus);
-  if (!nstreams(gin))
-    return 0;
+  gin = openstreams(inbase, incpus, ncpus, cpuid, modulus);
   go = goinit(width, modulus, nextx, ncpus, cpuid);
   sprintf(outbase,"%d.%d/yx.%02d.%02d",width,modidx,y+(nextx==0),nextx);
 
   nnewillcnt = 0LL;
   jts = jtalloc(msize, modulus, LOCBITS);
-  for (nin=0LL; (mb = minstream(gin))->state != FINALSTATE; nin++) {
-    sn.cnt = mb->cnt;
-    // printf("expanding %llo\n", mb->state);
-    nnew = expandstate(mb->state, x, newstates);
-    for (i=0; i<nnew; i++) {
-      sn.state = newstates[i];
-      //printf("inserting %llo\n", sn.state);
-      jtinsert(jts, &sn);
+  if (nstreams(gin)) {
+    for (nin=0LL; (mb = minstream(gin))->state != FINALSTATE; nin++) {
+      sn.cnt = mb->cnt;
+      // printf("expanding %llo\n", mb->state);
+      nnew = expandstate(mb->state, x, newstates);
+      for (i=0; i<nnew; i++) {
+        sn.state = newstates[i];
+        //printf("inserting %llo\n", sn.state);
+        jtinsert(jts, &sn);
+      }
+      if (nnew < 3) // nnew == 2
+        modadd(modulus, &nnewillcnt, mb->cnt);
+      if (jtfull(jts))
+        dumpstates(go, jts, outbase, noutfiles++, mb->state);
+      deletemin(gin);
     }
-    if (nnew < 3) // nnew == 2
-      modadd(modulus, &nnewillcnt, mb->cnt);
-    if (jtfull(jts))
-      dumpstates(go, jts, outbase, noutfiles++);
-    deletemin(gin);
   }
-  if (!jtempty(jts))
-    dumpstates(go, jts, outbase, noutfiles++);
+  dumpstates(go, jts, outbase, noutfiles++, FINALSTATE);
   jtfree(jts);
 
-  hidefiles(go, outbase, noutfiles);
-
   printf("(%d,%d) size %lu xsize %lu mod ",y,x,nin,totalread(gin));
   if (modulus)
     printf("%lu",modulus);

outstream.c

@@ -122,7 +122,7 @@ void writerecord(goout *go, uint64_t state, uint64_t cnt)
 {
   uint64_t delta = state - go->bufstate;
 
-  assert(delta > 0);
+  assert(!go->bufstate || delta > 0);
   writedelta(go, delta);
   if (!fwrite(&cnt, sizeof(uint64_t),1,go->fpdelta)) {
     printf("failed to write count\n");
@@ -152,20 +152,20 @@ void closedelta(goout *go)
   modadd(go->modulus, &go->cumlegal, go->nlegal);
 }
 
-void dumpstates(goout *go, jtset *jts, char *outbase, int extension)
+void dumpstates(goout *go, jtset *jts, char *outbase, int iteration, uint64_t state)
 {
   char formatname[FILENAMELEN];
   statecnt *sc;
   uint64_t prevstate;
   int i;
 
-  if (extension==0) {
+  if (iteration==0) {
     for (i=0; i<go->ncpus; i++) {
       sprintf(formatname,"%s/fromto.%d.%d", outbase, go->cpuid, i);
       mkdir(formatname, 0755);
     }
   }
-  sprintf(formatname,"%s/fromto.%d.%%d/%d", outbase, go->cpuid, extension);
+  sprintf(formatname,"%s/fromto.%d.%%d/%d.%lo", outbase, go->cpuid, iteration, state);
 //printf("formatname = %s\n", formatname);
   jtstartfor(jts, 3*go->width);
   sc = jtnext(jts);
@@ -174,7 +174,7 @@ void dumpstates(goout *go, jtset *jts, char *outbase, int extension)
     sprintf(go->outname,formatname, i);
     opendelta(go);
     while (sc != NULL && sc->state < go->parts[i]) {
-      assert(sc->state > prevstate);
+      assert(sc->state==STARTSTATE || sc->state > prevstate);
       prevstate = sc->state;
       writerecord(go, sc->state, sc->cnt);
       sc = jtnext(jts);
@@ -185,19 +185,6 @@ void dumpstates(goout *go, jtset *jts, char *outbase, int extension)
   assert(jtempty(jts));
 }
 
-void hidefiles(goout *go, char *basename, int extension)
-{
-  char outname[FILENAMELEN],bak[FILENAMELEN];
-  int i;
-
-  for (i=0; i<go->ncpus; i++) {
-    sprintf(outname,"%s/fromto.%d.%d/%d", basename, go->cpuid, i, extension);
-    sprintf(bak,"%s/fromto.%d.%d/%d.bak", basename, go->cpuid, i, extension);
-    if (rename(outname, bak) == 0)
-      printf("%s renamed to %s\n",outname, bak);
-  }
-}
-
 uint64_t needywritten(goout *go)
 {
   return go->cumneedy;
@@ -273,13 +260,11 @@ int main(int argc, char *argv[])
   while (fread(&sb.state,sizeof(uint64_t),2,stdin)==2) {
     jtinsert(jts, &sb);
     if (jtfull(jts))
-      dumpstates(go, jts, outbase, noutfiles++);
+      dumpstates(go, jts, outbase, noutfiles++, sb.state);
   }
-  if (!jtempty(jts))
-    dumpstates(go, jts, outbase, noutfiles++);
+  dumpstates(go, jts, outbase, noutfiles++, FINALSTATE);
   jtfree(jts);
 
-  hidefiles(go, outbase, noutfiles);
   if (nextx==0)
     printf("legal(%dx%d) %% %llu = %llu\n",y+1,width,modulus,legalwritten(go));
   return 0;

outstream.h

@@ -3,7 +3,7 @@
 typedef struct goout goout;
 
 goout *goinit(int width, uint64_t modulus, int bump, int ncpus,int cpuid);
-void dumpstates(goout *go, jtset *jts, char *outbase, int extension);
+void dumpstates(goout *go, jtset *jts, char *outbase, int iteration, uint64_t state);
 void hidefiles(goout *go, char *basename, int extension);
 uint64_t needywritten(goout *go);
 uint64_t legalwritten(goout *go);