// Copyright 1996 by Jon Dart.  All Rights Reserved.

// Stand-alone executable to build the binary opening book from
// a text file.

// The book file is a binary file consisting of one or more "pages".
// Each page is a self-contained data structure consisting of
// three portions: a header, a hash table, and an array of book
// entries.  The page size and hash table size are configurable.

// By default each page is 65536 bytes, with a hash table of 2043
// entries.  Each page can hold around 5,000 half-moves.

// The header is defined in bookdefs.h.  It currently contains 4
// fields:
//   version number
//   number of pages
//   page size (bytes)
//   hash table size (number of entries)

// The hash table is an array of unsigned indexes into the book
// entry array.  The high-order 32 bits of the hash code for the
// position modulo the hash table size is used to look up in this
// array the first book entry for the position.

// For the style book, the 4 bits of the hash table probe value
// (hash code % hash table size) are modified to equal the style
// value.  This is done both on construction of the book and on
// access to it, guaranteeing that only nodes which match the
// preferred style are accessed.

// The book entries themselves are structures defined in bookentr.h.
// Each one is 13 bytes and contains the full hash code, the move
// index, win/loss info for the move, and a weight indicating how
// often the computer should play the move.  Hash entries for each
// chain in the hash table are stored contiguously in the book, and
// the weight byte for the last entry in each chain has its hi bit set.

#include "stdafx.h"
#include "board.h"
#include "emove.h"
#include "bookentr.h"
#include "bookdefs.h"
#include "hash.h"
#include "bhash.h"
#include "movegen.h"
#include "notation.h"
#include "types.h"
#include "debug.h"
#include "bearing.h"
#include "globals.h"
extern "C"
{
#include <string.h>
#include <ctype.h>
};

#include <fstream.h>
#include <iostream.h>

static const int Buffer_Size = 4096;
// max. moves per page
static unsigned max_moves;

enum {White_Win, Black_Win, Draw, Unknown} last_result = Unknown;

// number of pages in the book.
static int bookPages;
// size of a single page
static int pageSize = 65536;
// hash table size (number of entries).  Each entry is 2 bytes
static unsigned int hashSize = 2043;

static int style = 0;

static char *output_name;

struct book_info
{
    unsigned next_free;
    CWordArray *hash_table; 
    CPtrArray *book_moves; // array of Book_Entry *
};

// During opening book construction, we add an extra member to
// each book entry type.
class Book_Entry2  : public Book_Entry
{
public:
     Book_Entry2( hash_t hc, byte rec, byte mv_indx,
                  unsigned int nxt, BOOL last = FALSE);

     unsigned int next;
     unsigned int count; // number of times position has occurred
     unsigned int white_wins; // wins for White
     unsigned int black_wins; // wins for Black
};

Book_Entry2::Book_Entry2( hash_t hc, byte rec, byte mv_indx,
                          unsigned int nxt, BOOL last)
:Book_Entry(hc,rec,mv_indx,last),next(nxt)
{
   white_wins = black_wins = 0;
   count = 1;
}

static book_info *book_data;

static Book_Entry2 *
search(const unsigned start, const unsigned move_index,
       const int page,
       const hash_t hc)
{
   unsigned indx = start;
   Book_Entry2 be(hc, 0, move_index, 0, FALSE);
   while (indx != INVALID)
   {
      UNCOND_ASSERT(indx < max_moves);
      Book_Entry2 *entry = (Book_Entry2*)(*(book_data[page].book_moves))[indx];
      if (entry->is_equal(be) && entry->move_index == move_index)
              return entry;
      indx = entry->next;
   }
   return NULL;
}

static void
add_move(const Board & board, int move_index, int recommend)
{
   unsigned page = (unsigned)(board.HashCode() % bookPages);
   // use the high order bits of the hash code as the table probe
   unsigned probe = (unsigned) (board.HashCode()>>32) % hashSize;
   if (style)
   {
       // Make the low-level bits of the bucket index match the
       // style
       probe &= ~(MAX_STYLES-1);
       probe |= style;
   }
   unsigned hit = (*(book_data[page].hash_table))[probe];
   Book_Entry2 *old_entry;
   if (old_entry = search(hit, move_index, page,
                                 board.HashCode()))
   {
      old_entry->count++;
      if (last_result == White_Win)
          old_entry->white_wins++;
      else if (last_result == Black_Win)
          old_entry->black_wins++;
   }
   else
   {
      Book_Entry2 *new_entry = new Book_Entry2(board.HashCode(), recommend,
                                             move_index, hit, FALSE);
      ASSERT(new_entry);
      if (book_data[page].next_free >= max_moves)
      {
         cerr << "\nError - Too many moves in book " << 
              "(page " << page << ")" << endl;
         exit(-1);
      }
      if (last_result == White_Win)
          new_entry->white_wins++;
      else if (last_result == Black_Win)
          new_entry->black_wins++;
      unsigned next = book_data[page].next_free;
      (*(book_data[page].book_moves))[next] = new_entry;
      (*(book_data[page].hash_table))[probe] = next;
#ifdef _TRACE
      cout << "pg:" << page << " p:" << probe << 
           " h:" << (hex) << Bitmap(board.HashCode()).hivalue() << 
           Bitmap(board.HashCode()).lovalue() <<
           (dec) << " i:" <<
           move_index << " f:" << next << " w:" << recommend << endl;
#endif
      book_data[page].next_free++;
   }
}

static void write_page(int page,ofstream &book_file)
{
   byte *write_buffer = new byte[Buffer_Size];
   ASSERT(write_buffer);

   // write the header
   struct Book_Header hdr;
   
   hdr.version = Book_Version;
   hdr.num_pages = bookPages;
   hdr.page_capacity = pageSize;
   hdr.hash_table_size = hashSize;
   char buf[Header_Size];
   //book_file.write((char*)&hdr, Header_Size);
   buf[0] = hdr.version;
   buf[1] = hdr.num_pages;
   memcpy(buf+2,&hdr.page_capacity,4);
   memcpy(buf+6,&hdr.hash_table_size,4);
   book_file.write(buf,Header_Size);

   // Re-work the hash table.  Right now it holds pointers to the
   // head of each hash chain, linked together with "next" pointers.
   // We are going to rearrange the chains so they are contiguous
   // in the book.  Then the hash table will hold indexes to the start
   // of each block of hash entries - these indexes are counts of
   // how many Book_Entry objects lie before the start of the desired
   // block.

   int running_count = 0;
   for (unsigned i = 0; i < hashSize; i++)
   {
      unsigned ht_entry = (*(book_data[page].hash_table))[i];
      if (ht_entry == INVALID)
      {
         book_file.put((char)(INVALID % 256));
         book_file.put((char)(INVALID / 256));
      }
      else
      {
         book_file.put((char)(running_count % 256));
         book_file.put((char)(running_count / 256));

         unsigned indx = ht_entry;
         unsigned weight_count = 0;
         Book_Entry2 *entry = NULL;
         while (indx != INVALID)
         {
           entry = (Book_Entry2*)(*(book_data[page].book_moves))[indx];
           if (entry->hash_code() & 1 == 0)
           {
              // white is on move
              entry->set_winloss((((int)entry->white_wins-(int)entry->black_wins)*100)/(int)entry->count);
           }
           else
           {
              entry->set_winloss((((int)entry->black_wins-(int)entry->white_wins)*100)/(int)entry->count);
           }
           int i = 1;
           if (entry->get_winloss() < -100)
               i = 0;
           entry->set_frequency((uint16)entry->count);
           indx = entry->next;
           ++running_count;
         }
         if (entry)
           entry->set_last();
      }
   }

   // write the book moves

   unsigned index = 0;
   for (i = 0; i < hashSize; i++)
   {
      unsigned ht_entry = (*(book_data[page].hash_table))[i];
      while (ht_entry != INVALID)
      {
         Book_Entry2 *book_entr = 
             (Book_Entry2*)(*(book_data[page].book_moves))[ht_entry];
         if (index + Entry_Size <= Buffer_Size)
         {
                memcpy(write_buffer + index,
                           &book_entr->my_hash_code, Entry_Size);
                index += Entry_Size;
         } 
         else
         {
                unsigned to_go = Buffer_Size - index;
                static byte buf[Entry_Size];
                memcpy(buf,&book_entr->my_hash_code,Entry_Size);
                if (to_go)
                {
                   memcpy(write_buffer + index,buf,to_go);
                }
                book_file.write(write_buffer, Buffer_Size);
                memcpy(write_buffer, buf + to_go, Entry_Size - to_go);
                index = Entry_Size - to_go;
         }
         ht_entry = book_entr->next;
      }
   }
   if (index)
      book_file.write(write_buffer, index);
   long bookSize = book_file.tellp();
   while (bookSize % pageSize != 0L)
   {
      book_file.put('\0');
      bookSize++;
   }
   delete[] write_buffer;

   cout << "page " << page << ": " <<
      book_data[page].next_free << " moves." << endl;
}

static void write_book()
{
   ofstream book_file(output_name,
                       ios::out | ios::trunc | ios::binary);
   long total_moves = 0L;
   for (int i = 0; i < bookPages && book_file.good(); i++)
   {
      write_page(i,book_file);
      total_moves += book_data[i].next_free;
   }
   cout << total_moves << " total moves." << endl;
   book_file.close();
}

int __cdecl
main(int argc, char **argv)
{
   char book_name[MAX_PATH];
   fstream infile;
   
   init_globals(argv[0],FALSE,FALSE);
   atexit(cleanup_globals);

   output_name = NULL;
   int arg = 1;
   while (arg < argc)
   {
      if (*argv[arg] == '-')
      {
         char c = argv[arg][1];
         switch(c)
         {
         case 'p': /* page size */
           ++arg;
           pageSize = atoi(argv[arg]);
           if (pageSize == 0)
           {
              cerr << "Illegal page size" << endl;
              exit(-1);
           }
           break;
         case 'o': /* output file name */
           ++arg;
           output_name = strdup(argv[arg]);
           break;
         case 'n': /* number of pages (power of 2)*/
           ++arg;
           bookPages = atoi(argv[arg]);
           if (bookPages == 0)
           {
              cerr << "Illegal size specified for -n" << endl;
              exit(-1);
           }
           break;
        case 'h': /* hash size */         
           ++arg;
           hashSize = atoi(argv[arg]);
           if (hashSize == 0)
           {
              cerr << "Illegal hash size" << endl;
              exit(-1);
           }
           break;
         default:
           cerr << "Illegal switch: " << c << endl;
           cerr << "Usage:" << endl;
           cerr << "makebook -n <bits> -p <page size> -h <hash size>" << endl;
           cerr << "         -o <output file> <input file>" << endl;
           exit(-1);
         }
         ++arg;
      }
      else
          break;
   }
   max_moves = (pageSize - hashSize*2 - Header_Size)/Entry_Size;
   if (max_moves < 5)
   {
      cerr << "Insufficient page size.  Use a larger value for -p"
          << endl;
      exit(-1);
   }
   else if (max_moves*Entry_Size < pageSize/4)
   {
      cerr << "Warning: hash table will consume > 1/4 of page." << endl;
      cerr << "Suggest you use a larger page size or smaller hash table size." << endl;
   }

   if (arg < argc)
      strcpy(book_name, argv[arg]);
   else
      strcpy(book_name, "book.txt");
   if (!output_name)
   {
      output_name = (char*)strdup("BOOK.BIN");
   }
   // Initialize the "book_data" structures that will hold
   // the opening book info during book construction.
   
   book_data = new book_info[bookPages];
   for (int i = 0; i < bookPages; i++)
   {
      book_data[i].book_moves = new CPtrArray ();
      book_data[i].book_moves->SetSize(max_moves);
      book_data[i].hash_table = new CWordArray ();
      book_data[i].hash_table->SetSize(hashSize);
      book_data[i].next_free = 0;
   }
   
   for (int j = 0; j < bookPages; j++)
   {
      for (unsigned i = 0; i < hashSize; i++)
      {
         (*(book_data[j].hash_table))[i] = INVALID;
      }
      for (i = 0; i < max_moves; i++)
      {
         (*(book_data[j].book_moves))[i] = NULL;
      }
   }
   int dots = 0;
   while (arg < argc)
   {
   strcpy(book_name, argv[arg++]);
   infile.open(book_name, ios::in);
   if (!infile.good())
   {
      cerr << "Can't open book file: " << argv[arg] << endl;
      return -1;
   }

   char buf[256];
   char movebuf[20];
   unsigned line = 0;
   char *q;
   Board board;
   Board tmpboard;
   int recommend;
   Move moves[Move_Generator::MaxMoves];

   while (!infile.eof())
   {
      recommend = 50;
      infile.getline(buf, 256);
#ifdef _TRACE
      cout << buf << endl;
#endif
      line++;
      char *p = buf;
      while (isspace(*p))
             p++;
      switch (*p)
      {
      case '\0':
            continue;
      case ';':
            {
               if (strstr(p,"1/2-1/2"))
                   last_result = Draw;
               else if (strstr(p,"0-1") || strstr(p,"0:1"))
                   last_result = Black_Win;
               else if (strstr(p,"1-0") || strstr(p,"1:0"))
                   last_result = White_Win;
               else 
                   last_result = Unknown;
               continue;
            }
      case 'm':
            tmpboard = board;
            continue;
      case 'r':
            last_result = Unknown;
            board = tmpboard;
            continue;
      case 's':
            ++p;
            while (isspace(*p))
                    p++;
            if (*p && isdigit(*p))
            {
                 sscanf(p,"%d",&style);
            }
            while (!isspace(*p)) ++p;
            
            continue;
      case '-':
            last_result = Unknown;
            board.Reset();
            break;
      default:
          {
            while (*p)
            {
               while (isspace(*p))
                      p++;
               if (*p == '\0')
                  break;
               q = movebuf;
               int count = 0;
               while (!isspace(*p) && *p && count < 19)
               {
                      *q++ = *p++;
                      ++count;
               }
               *q = '\0';
               Move move = Notation::Value(board, board.Side(), movebuf);
               if (move.IsNull())
               {
                      cerr << endl << "Illegal move in file, line " << 
                         line << " (" << movebuf << ")" << endl;
                      infile.close();
                      return -1;
               }
               ExtendedMove emove(board, move);

               // check (pseudo) legality:
               Move_Generator mg(board, 0, Move::NullMove());
               int found = 0;
               int move_indx = 0;

               int n = mg.Generate_Moves(moves, TRUE /* repeatable */);

               for (int i = 0; i < n; i++)
               {
                      if (moves[i] == emove)
                      {
                         move_indx = i;
                         found++;
                         break;
                      }
               }
               if (!found)
               {
                       cerr << endl << "Illegal move in file, line " << line << " ("
                          << movebuf << ")" << endl;
                       infile.close();
                       return -1;
               }
               dots++;
               if (dots % 50 == 0)
                       cout << '.' << (flush);
               if (dots > 3000)
               {
                       cout << endl;
                       dots = 1;
               }
               recommend = 50;
               while (isspace(*p))
                       p++;
               if (*p)
               {
                       if ((*p >= '0') && (*p <= '9'))
                       {
                           recommend = 10*(*p - '0');
                           p++;
                       }
               }
               add_move(board, move_indx, recommend);
               board.MakeMove(emove);
            }
          }
      }
   }
   infile.close();
   }

   cout << endl;
   write_book();
   cout << "Total book size: " <<
       pageSize*bookPages/1024 << "K. ";
   cout << "Capacity is about " <<
       max_moves*bookPages << " moves." << endl;
   
   return 0;
}
