Changeset 6449

Timestamp:

Jan 17, 2010, 11:49:48 AM (15 years ago)

Author:

scheusso

Message:

changed diff behaviour in order to reduce datasize before and after compress
this reduces time needed for gamestate diff and compress about 50%

Location:

code/branches/network2/src/libraries/network

Files:

• GamestateClient.cc (modified) (1 diff)
• GamestateManager.cc (modified) (7 diffs)
• GamestateManager.h (modified) (1 diff)
• packet/Gamestate.cc (modified) (23 diffs)
• packet/Gamestate.h (modified) (3 diffs)
• synchronisable/Serialise.h (modified) (1 diff)
• synchronisable/Synchronisable.cc (modified) (13 diffs)
• synchronisable/Synchronisable.h (modified) (8 diffs)

code/branches/network2/src/libraries/network/GamestateClient.cc

@@ -161 +161 @@
       assert(b);
     }
-    if(gs->isDiffed()){
-      packet::Gamestate *base = gamestateMap_[gs->getBaseID()];
-      if(!base){
-        COUT(3) << "could not find base gamestate id: " << gs->getBaseID() << endl;
-        delete gs;
-        return 0;
-      }
-//       assert(base); //TODO: fix this
-      packet::Gamestate *undiffed = gs->undiff(base);
-      delete gs;
-      gs=undiffed;
-      COUT(5) << "successfully undiffed gamestate id: " << undiffed->getID() << std::endl;
-    }
+//     if(gs->isDiffed()){
+//       packet::Gamestate *base = gamestateMap_[gs->getBaseID()];
+//       if(!base){
+//         COUT(3) << "could not find base gamestate id: " << gs->getBaseID() << endl;
+//         delete gs;
+//         return 0;
+//       }
+// //       assert(base); //TODO: fix this
+//       packet::Gamestate *undiffed = gs->undiff(base);
+//       delete gs;
+//       gs=undiffed;
+//       COUT(5) << "successfully undiffed gamestate id: " << undiffed->getID() << std::endl;
+//     }
     if(gs->spreadData(0x2))
       return gs;

code/branches/network2/src/libraries/network/GamestateManager.cc

@@ -43 +43 @@
 #include <cassert>
 #include <queue>
+#include "util/Clock.h"
 // #include <boost/thread/mutex.hpp>
 
@@ -158 +159 @@
 
       clientGamestates.push(0);
-      finishGamestate( cid, &clientGamestates.back(), client, reference );
+      finishGamestate( cid, clientGamestates.back(), client, reference );
       //FunctorMember<GamestateManager>* functor =
 //       ExecutorMember<GamestateManager>* executor = createExecutor( createFunctor(&GamestateManager::finishGamestate, this) );
@@ -180 +181 @@
 
 
-  void GamestateManager::finishGamestate( unsigned int clientID, packet::Gamestate** destgamestate, packet::Gamestate* base, packet::Gamestate* gamestate ) {
+  void GamestateManager::finishGamestate( unsigned int clientID, packet::Gamestate*& destgamestate, packet::Gamestate* base, packet::Gamestate* gamestate ) {
     //why are we searching the same client's gamestate id as we searched in
     //Server::sendGameState?
@@ -186 +187 @@
     //chose wheather the next gamestate is the first or not
 
-    packet::Gamestate *gs = gamestate->doSelection(clientID, 20000);
-//     packet::Gamestate *gs = new packet::Gamestate(*gamestate);
+//     packet::Gamestate *gs = gamestate->doSelection(clientID, 20000);
+//       packet::Gamestate* gs = new packet::Gamestate(*gamestate);
+//     packet::Gamestate* gs = gamestate;
+    packet::Gamestate *gs = new packet::Gamestate(*gamestate);
 //     packet::Gamestate *gs = new packet::Gamestate();
 //     gs->collectData( id_, 0x1 );
@@ -193 +196 @@
     gamestateMap_[clientID][gamestate->getID()]=gs;
 //     this->threadMutex_->unlock();
+      Clock clock;
+      clock.capture();
 
     if(base)
@@ -200 +205 @@
 //       packet::Gamestate* gs1  = gs;
       packet::Gamestate *diffed = gs->diff(base);
+      if( diffed->getDataSize() == 0 )
+      {
+        delete diffed;
+        destgamestate = 0;
+        return;
+      }
+      else
+        gs = diffed;
       //packet::Gamestate *gs2 = diffed->undiff(gs);
 //       assert(*gs == *gs2);
-      gs = diffed;
 //       packet::Gamestate* gs2 = gs->undiff(client);
 //       gs = new packet::Gamestate(*gs);
@@ -214 +226 @@
     bool b = gs->compressData();
     assert(b);
-//     COUT(4) << "sending gamestate with id " << gs->getID();
+      clock.capture();
+      COUT(0) << "diff time: " << clock.getDeltaTime() << endl;
+//     COUT(5) << "sending gamestate with id " << gs->getID();
 //     if(gamestate->isDiffed())
-//     COUT(4) << " and baseid " << gs->getBaseID() << endl;
+//       COUT(5) << " and baseid " << gs->getBaseID() << endl;
 //     else
-//     COUT(4) << endl;
+//       COUT(5) << endl;
     gs->setClientID(clientID);
-    *destgamestate = gs;
+    destgamestate = gs;
   }
 

code/branches/network2/src/libraries/network/GamestateManager.h

@@ -76 +76 @@
     void sendGamestates();
 //     packet::Gamestate *popGameState(unsigned int clientID);
-    void finishGamestate( unsigned int clientID, packet::Gamestate** destgamestate, packet::Gamestate* base, packet::Gamestate* gamestate );
+    void finishGamestate( unsigned int clientID, packet::Gamestate*& destgamestate, packet::Gamestate* base, packet::Gamestate* gamestate );
 
     bool getSnapshot();

code/branches/network2/src/libraries/network/packet/Gamestate.cc

@@ -45 +45 @@
 #define PACKET_FLAG_GAMESTATE  PacketFlag::Reliable
 
-
-Gamestate::Gamestate()
+inline bool memzero( uint8_t* data, uint32_t datalength)
+{
+  uint64_t* d = (uint64_t*)data;
+
+  for( unsigned int i=0; i<datalength/8; i++ )
+  {
+    if( *(d+i) != 0 )
+      return false;
+  }
+  // now process the rest (when datalength isn't a multiple of 4)
+  for( unsigned int j = 8*(datalength/8); j<datalength; j++ )
+  {
+    if( *(data+j) != 0 )
+      return false;
+  }
+  return true;
+}
+
+
+Gamestate::Gamestate():
+  header_(0)
 {
   flags_ = flags_ | PACKET_FLAG_GAMESTATE;
-  header_ = 0;
-}
+}
+
 
 Gamestate::Gamestate(uint8_t *data, unsigned int clientID):
-    Packet(data, clientID)
+  Packet(data, clientID)
 {
   flags_ = flags_ | PACKET_FLAG_GAMESTATE;
@@ -59 +78 @@
 }
 
+
 Gamestate::Gamestate(uint8_t *data)
 {
   flags_ = flags_ | PACKET_FLAG_GAMESTATE;
-  data_=data;
+  data_ = data;
   header_ = new GamestateHeader(data_);
 }
 
+
 Gamestate::Gamestate(const Gamestate& g) :
-    Packet( *(Packet*)&g )
+    Packet( *(Packet*)&g ), nrOfVariables_(0)
 {
   flags_ = flags_ | PACKET_FLAG_GAMESTATE;
   header_ = new GamestateHeader(data_);
+  sizes_ = g.sizes_;
 }
 
@@ -79 +101 @@
     delete header_;
 }
+
 
 bool Gamestate::collectData(int id, uint8_t mode)
@@ -91 +114 @@
     return false;
   data_ = new uint8_t[size + GamestateHeader::getSize()];
-  if(!data_){
+  if(!data_)
+  {
     COUT(2) << "GameStateManager: could not allocate memory" << std::endl;
     return false;
@@ -101 +125 @@
 
   //start collect data synchronisable by synchronisable
-  uint8_t *mem=data_;
+  uint8_t *mem = data_; // in this stream store all data of the variables and the headers of the synchronisable
   mem += GamestateHeader::getSize();
   ObjectList<Synchronisable>::iterator it;
-  for(it = ObjectList<Synchronisable>::begin(); it; ++it){
+  for(it = ObjectList<Synchronisable>::begin(); it; ++it)
+  {
 
 //     tempsize=it->getSize(id, mode);
 
-    tempsize = it->getData(mem, id, mode);
+    tempsize = it->getData(mem, this->sizes_, id, mode);
     if ( tempsize != 0 )
       dataVector_.push_back( obj(it->getObjectID(), it->getCreatorID(), tempsize, mem-data_) );
 
 #ifndef NDEBUG
-    if(currentsize+tempsize > size){
+    if(currentsize+tempsize > size)
+    {
       assert(0); // if we don't use multithreading this part shouldn't be neccessary
       // start allocate additional memory
@@ -148 +174 @@
 }
 
+
 bool Gamestate::spreadData(uint8_t mode)
 {
@@ -153 +180 @@
   assert(data_);
   assert(!header_->isCompressed());
-  assert(!header_->isDiffed());
   uint8_t *mem=data_+GamestateHeader::getSize();
+  bool diffed = header_->isDiffed();
   Synchronisable *s;
 
   // update the data of the objects we received
-  while(mem < data_+GamestateHeader::getSize()+header_->getDataSize()){
+  while(mem < data_+GamestateHeader::getSize()+header_->getDataSize())
+  {
     SynchronisableHeader objectheader(mem);
 
@@ -166 +194 @@
       if (!GameMode::isMaster())
       {
-        Synchronisable::fabricate(mem, mode);
+        Synchronisable::fabricate(mem, diffed, mode);
       }
       else
       {
-        mem += objectheader.getDataSize();
+        mem += objectheader.getDataSize()+SynchronisableHeader::getSize();
       }
     }
@@ -181 +209 @@
    // In debug mode, check first, whether there are no duplicate objectIDs
 #ifndef NDEBUG
-  if(this->getID()%1000==0){
+  if(this->getID()%1000==1)
+  {
     std::list<uint32_t> v1;
     ObjectList<Synchronisable>::iterator it;
-    for (it = ObjectList<Synchronisable>::begin(); it != ObjectList<Synchronisable>::end(); ++it) {
-      if (it->getObjectID() == OBJECTID_UNKNOWN) {
-        if (it->objectMode_ != 0x0) {
+    for (it = ObjectList<Synchronisable>::begin(); it != ObjectList<Synchronisable>::end(); ++it)
+    {
+      if (it->getObjectID() == OBJECTID_UNKNOWN)
+      {
+        if (it->objectMode_ != 0x0)
+        {
           COUT(0) << "Found object with OBJECTID_UNKNOWN on the client with objectMode != 0x0!" << std::endl;
           COUT(0) << "Possible reason for this error: Client created a synchronized object without the Server's approval." << std::endl;
@@ -193 +225 @@
         }
       }
-      else {
+      else
+      {
         std::list<uint32_t>::iterator it2;
-        for (it2 = v1.begin(); it2 != v1.end(); ++it2) {
-          if (it->getObjectID() == *it2) {
+        for (it2 = v1.begin(); it2 != v1.end(); ++it2)
+        {
+          if (it->getObjectID() == *it2)
+          {
             COUT(0) << "Found duplicate objectIDs on the client!" << std::endl
                     << "Are you sure you don't create a Sychnronisable objcect with 'new' \
@@ -211 +246 @@
 }
 
+
 uint32_t Gamestate::getSize() const
 {
@@ -222 +258 @@
 }
 
-bool Gamestate::operator==(packet::Gamestate gs){
+
+bool Gamestate::operator==(packet::Gamestate gs)
+{
   uint8_t *d1 = data_+GamestateHeader::getSize();
   uint8_t *d2 = gs.data_+GamestateHeader::getSize();
@@ -233 +271 @@
 }
 
+
 bool Gamestate::process()
 {
   return GamestateHandler::addGamestate(this, getClientID());
 }
-
 
 
@@ -253 +291 @@
   int retval;
   retval = compress( dest, &buffer, source, (uLong)(header_->getDataSize()) );
-  switch ( retval ) {
+  switch ( retval )
+  {
     case Z_OK: COUT(5) << "G.St.Man: compress: successfully compressed" << std::endl; break;
     case Z_MEM_ERROR: COUT(1) << "G.St.Man: compress: not enough memory available in gamestate.compress" << std::endl; return false;
@@ -270 +309 @@
   header_->setCompSize( buffer );
   header_->setCompressed( true );
-  COUT(5) << "gamestate compress datasize: " << header_->getDataSize() << " compsize: " << header_->getCompSize() << std::endl;
+  COUT(0) << "gamestate compress datasize: " << header_->getDataSize() << " compsize: " << header_->getCompSize() << std::endl;
   return true;
 }
+
+
 bool Gamestate::decompressData()
 {
@@ -289 +330 @@
   uLongf length=bufsize;
   retval = uncompress( dest, &length, source, (uLong)compsize );
-  switch ( retval ) {
+  switch ( retval )
+  {
     case Z_OK: COUT(5) << "successfully decompressed" << std::endl; break;
     case Z_MEM_ERROR: COUT(1) << "not enough memory available" << std::endl; return false;
@@ -301 +343 @@
   delete temp;
 
-  if (this->bDataENetAllocated_){
+  if (this->bDataENetAllocated_)
+  {
     // Memory was allocated by ENet. --> We let it be since enet_packet_destroy will
     // deallocated it anyway. So data and packet stay together.
     this->bDataENetAllocated_ = false;
   }
-  else{
+  else
+  {
     // We allocated the memory in the first place (unlikely). So we destroy the old data
     // and overwrite it with the new decompressed data.
@@ -320 +364 @@
 }
 
-/*Gamestate *Gamestate::diff(Gamestate *base)
-{
-  assert(data_);
-  assert(!header_->isCompressed());
-  assert(!header_->isDiffed());
-  GamestateHeader diffHeader(base->data_);
-  uint8_t *basep = GAMESTATE_START(base->data_), *gs = GAMESTATE_START(this->data_);
-  uint32_t of=0; // pointers offset
-  uint32_t dest_length=0;
-  dest_length=header_->getDataSize();
-  if(dest_length==0)
-    return NULL;
-  uint8_t *ndata = new uint8_t[dest_length*sizeof(uint8_t)+GamestateHeader::getSize()];
-  uint8_t *dest = ndata + GamestateHeader::getSize();
-  while(of < diffHeader.getDataSize() && of < header_->getDataSize()){
-    *(dest+of)=*(basep+of)^*(gs+of); // do the xor
-    ++of;
-  }
-  if(diffHeader.getDataSize()!=header_->getDataSize()){
-    uint8_t n=0;
-    if(diffHeader.getDataSize() < header_->getDataSize()){
-      while(of<dest_length){
-        *(dest+of)=n^*(gs+of);
-        of++;
-      }
-    }
-  }
-
-  Gamestate *g = new Gamestate(ndata, getClientID());
-  *(g->header_) = *header_;
-  g->header_->setDiffed( true );
-  g->header_->setBaseID( base->getID() );
-  g->flags_=flags_;
-  g->packetDirection_ = packetDirection_;
-  return g;
-}*/
 
 Gamestate *Gamestate::diff(Gamestate *base)
@@ -363 +371 @@
   assert(!header_->isDiffed());
 
-  uint8_t *basep = GAMESTATE_START(base->data_);
-  uint8_t *gs = GAMESTATE_START(this->data_);
-  uint32_t dest_length = header_->getDataSize();
-
-  if(dest_length==0)
-    return NULL;
-
-  uint8_t *ndata = new uint8_t[dest_length*sizeof(uint8_t)+GamestateHeader::getSize()];
-  uint8_t *dest = GAMESTATE_START(ndata);
-
-  rawDiff( dest, gs, basep, header_->getDataSize(), base->header_->getDataSize() );
-#ifndef NDEBUG
-  uint8_t *dest2 = new uint8_t[dest_length];
-  rawDiff( dest2, dest, basep, header_->getDataSize(), base->header_->getDataSize() );
-  assert( memcmp( dest2, gs, dest_length) == 0 );
-  delete dest2;
-#endif
-
-  Gamestate *g = new Gamestate(ndata, getClientID());
+
+  // *** first do a raw diff of the two gamestates
+
+  uint8_t *baseData = GAMESTATE_START(base->data_);
+  uint8_t *origData = GAMESTATE_START(this->data_);
+  uint32_t origLength = header_->getDataSize();
+  uint32_t baseLength = base->header_->getDataSize();
+
+  assert( origLength && baseLength );
+
+  COUT(0) << "newSize: " << origLength + GamestateHeader::getSize() + sizeof(uint32_t)*this->nrOfVariables_ << endl;
+  uint8_t *nData = new uint8_t[origLength + GamestateHeader::getSize() + sizeof(uint32_t)*this->nrOfVariables_]; // this is the maximum size needed in the worst case
+  uint8_t *dest = GAMESTATE_START(nData);
+
+  uint32_t baseOffset = 0; //offset in the diffed stream
+  uint32_t origOffset = 0; //offset in the new stream with removed 0's
+  std::vector<uint32_t>::iterator sizes = this->sizes_.begin();
+
+  while( origOffset < origLength )
+  {
+    //iterate through all objects
+
+    SynchronisableHeader h(origData+origOffset);
+
+    // Find (if possible) the current object in the datastream of the old gamestate
+    // Start at the current offset position
+    if(baseOffset >= baseLength)
+      baseOffset = 0;
+    uint8_t* temp = baseData + baseOffset;
+    uint32_t objectID = h.getObjectID();
+    assert(temp < baseData+baseLength);
+    assert(dest < nData + origLength + GamestateHeader::getSize() + sizeof(uint32_t)*this->nrOfVariables_);
+    assert(sizes != this->sizes_.end());
+    while ( temp < baseData+baseLength )
+    {
+      SynchronisableHeader htemp(temp);
+      if ( htemp.getObjectID() == objectID )
+      {
+        assert( h.getClassID() == htemp.getClassID() );
+        goto DODIFF;
+      }
+      temp += htemp.getDataSize()+SynchronisableHeader::getSize();
+    }
+    // If not found start looking at the beginning
+    temp = baseData;
+    while ( temp < baseData+baseOffset )
+    {
+      SynchronisableHeader htemp(temp);
+      if ( htemp.getObjectID() == objectID )
+      {
+        assert( h.getClassID() == htemp.getClassID() );
+        goto DODIFF;
+      }
+      temp += htemp.getDataSize()+SynchronisableHeader::getSize();
+    }
+    // Object is new, thus never transmitted -> just copy over
+    goto DOCOPY;
+
+
+DODIFF:
+    {
+//       if(baseOffset==0)
+//       {
+//         assert(origOffset==0);
+//       }
+      uint32_t objectOffset = SynchronisableHeader::getSize(); // offset inside the object in the origData and baseData
+      // Check whether the whole object stayed the same
+      if( memcmp( origData+origOffset+objectOffset, temp+objectOffset, h.getDataSize()) == 0 )
+      {
+        origOffset += objectOffset+ h.getDataSize(); // skip the whole object
+        baseOffset = temp + h.getDataSize()+SynchronisableHeader::getSize() - baseData;
+        sizes += Synchronisable::getSynchronisable(h.getObjectID())->getNrOfVariables();
+      }
+      else
+      {
+//         COUT(4) << "diff " << h.getObjectID() << ":";
+        // Now start to diff the Object
+        SynchronisableHeader h2(dest);
+        h2 = h; // copy over the objectheader
+        uint32_t variableID = 0;
+        uint32_t newObjectOffset = SynchronisableHeader::getSize();
+        // iterate through all variables
+        while( objectOffset < h.getDataSize()+SynchronisableHeader::getSize() )
+        {
+          // check whether variable changed and write id and copy over variable to the new stream
+          // otherwise skip variable
+          assert(sizes != this->sizes_.end());
+          uint32_t varSize = *sizes;
+          assert( varSize == Synchronisable::getSynchronisable(h.getObjectID())->getVarSize(variableID) );
+          if ( varSize != 0 )
+          {
+            if ( memcmp(origData+origOffset+objectOffset, temp+objectOffset, varSize) != 0 )
+            {
+//               COUT(4) << " c" << varSize;
+              *(uint32_t*)(dest+newObjectOffset) = variableID; // copy over the variableID
+              newObjectOffset += sizeof(uint32_t);
+              memcpy( dest+newObjectOffset, origData+origOffset+objectOffset, varSize );
+              newObjectOffset += varSize;
+              objectOffset += varSize;
+            }
+            else
+            {
+//               COUT(4) << " s" << varSize;
+              objectOffset += varSize;
+            }
+          }
+
+          ++variableID;
+          ++sizes;
+        }
+        if( Synchronisable::getSynchronisable(h.getObjectID())->getNrOfVariables() != variableID )
+          sizes += Synchronisable::getSynchronisable(h.getObjectID())->getNrOfVariables() - variableID;
+//         COUT(4) << endl;
+        h2.setDiffed(true);
+        h2.setDataSize(newObjectOffset-SynchronisableHeader::getSize());
+        assert(objectOffset == h.getDataSize()+SynchronisableHeader::getSize());
+        origOffset += objectOffset;
+        baseOffset += temp + h.getDataSize()+SynchronisableHeader::getSize() - baseData;
+        dest += newObjectOffset;
+      }
+
+      continue;
+    }
+
+DOCOPY:
+    {
+      // Just copy over the whole Object
+      memcpy( dest, origData+origOffset, h.getDataSize()+SynchronisableHeader::getSize() );
+      dest += h.getDataSize()+SynchronisableHeader::getSize();
+      origOffset += h.getDataSize()+SynchronisableHeader::getSize();
+      assert( Synchronisable::getSynchronisable(h.getObjectID()) );
+//       COUT(4) << "copy " << h.getObjectID() << endl;
+//       COUT(4) << "copy " << h.getObjectID() << ":";
+      //sizes += Synchronisable::getSynchronisable(h.getObjectID())->getNrOfVariables();
+      for( unsigned int i = 0; i < Synchronisable::getSynchronisable(h.getObjectID())->getNrOfVariables(); ++i )
+      {
+//         COUT(4) << " " << *sizes;
+        ++sizes;
+      }
+//       COUT(4) << endl;
+      assert(sizes != this->sizes_.end() || origOffset>=origLength);
+      continue;
+    }
+  }
+
+
+  Gamestate *g = new Gamestate(nData, getClientID());
   assert(g->header_);
   *(g->header_) = *header_;
   g->header_->setDiffed( true );
   g->header_->setBaseID( base->getID() );
+  g->header_->setDataSize(dest - nData - GamestateHeader::getSize());
   g->flags_=flags_;
   g->packetDirection_ = packetDirection_;
@@ -393 +530 @@
 }
 
-Gamestate *Gamestate::undiff(Gamestate *base)
-{
-  assert(this && base); assert(data_ && base->data_);
-  assert(!header_->isCompressed() && !base->header_->isCompressed());
-  assert(header_->isDiffed());
-
-  uint8_t *basep = GAMESTATE_START(base->data_);
-  uint8_t *gs = GAMESTATE_START(this->data_);
-  uint32_t dest_length = header_->getDataSize();
-
-  if(dest_length==0)
-    return NULL;
-
-  uint8_t *ndata = new uint8_t[dest_length*sizeof(uint8_t)+GamestateHeader::getSize()];
-  uint8_t *dest = ndata + GamestateHeader::getSize();
-
-  rawDiff( dest, gs, basep, header_->getDataSize(), base->header_->getDataSize() );
-
-  Gamestate *g = new Gamestate(ndata, getClientID());
-  assert(g->header_);
-  *(g->header_) = *header_;
-  g->header_->setDiffed( false );
-  g->flags_=flags_;
-  g->packetDirection_ = packetDirection_;
-  assert(!g->isDiffed());
-  assert(!g->isCompressed());
-  return g;
-}
-
-
-// Gamestate *Gamestate::diff(Gamestate *base)
-// {
-//   assert(data_);
-//   assert(!header_->isCompressed());
-//   assert(!header_->isDiffed());
-//   GamestateHeader diffHeader(base->data_);
-//   uint8_t *basep = GAMESTATE_START(base->data_), *gs = GAMESTATE_START(this->data_);
-//   uint32_t of=0; // pointers offset
-//   uint32_t dest_length=0;
-//   dest_length=header_->getDataSize();
-//   if(dest_length==0)
-//     return NULL;
-//   uint8_t *ndata = new uint8_t[dest_length*sizeof(uint8_t)+GamestateHeader::getSize()];
-//   uint8_t *dest = ndata + GamestateHeader::getSize();
-//
-//
-//   // LOOP-UNROLLED DIFFING
-//   uint32_t *dest32 = (uint32_t*)dest, *base32 = (uint32_t*)basep, *gs32 = (uint32_t*)gs;
-//   // diff in 4-byte steps
-//   while( of < (uint32_t)(header_->getDataSize())/4 ){
-//     if( of < (uint32_t)(diffHeader.getDataSize())/4 )
-//     {
-//       *(dest32+of)=*(base32+of) ^ *(gs32+of); // do the xor
-//       ++of;
-//     }
-//     else
-//     {
-//       *(dest32+of)=*(gs32+of); // same as 0 ^ *(gs32+of)
-//       ++of;
-//     }
-//   }
-//   for( unsigned int of2 = 0; of2 < header_->getDataSize()%4; ++of2 )
-//   {
-//     if( of*4+of2 < diffHeader.getDataSize() )
-//     {
-//       *(dest+4*of+of2)=*(basep+4*of+of2) ^ *(gs+4*of+of2); // do the xor
-//     }
-//     else
-//     {
-//       *(dest+4*of+of2)=*(gs+4*of+of2); // same as 0 ^ *(gs32+of)
-//     }
-//   }
-//
-//   Gamestate *g = new Gamestate(ndata, getClientID());
-//   *(g->header_) = *header_;
-//   g->header_->setDiffed( true );
-//   g->header_->setBaseID( base->getID() );
-//   g->flags_=flags_;
-//   g->packetDirection_ = packetDirection_;
-//   return g;
-// }
-
-
-void Gamestate::rawDiff( uint8_t* newdata, uint8_t* data, uint8_t* basedata, uint32_t datalength, uint32_t baselength)
-{
-  uint64_t* gd = (uint64_t*)data;
-  uint64_t* bd = (uint64_t*)basedata;
-  uint64_t* nd = (uint64_t*)newdata;
-
-  unsigned int i;
-  for( i=0; i<datalength/8; i++ )
-  {
-    if( i<baselength/8 )
-      *(nd+i) = *(gd+i) ^ *(bd+i);  // xor the data
-    else
-      *(nd+i) = *(gd+i); // just copy over the data
-  }
-  unsigned int j;
-  // now process the rest (when datalength isn't a multiple of 4)
-  for( j = 8*(datalength/8); j<datalength; j++ )
-  {
-    if( j<baselength )
-      *(newdata+j) = *(data+j) ^ *(basedata+j); // xor
-    else
-      *(newdata+j) = *(data+j); // just copy
-  }
-  assert(j==datalength);
-}
 
 Gamestate* Gamestate::doSelection(unsigned int clientID, unsigned int targetSize){
@@ -529 +558 @@
 //   COUT(0) << "myvector contains:";
 //   for ( itt=dataVector_.begin() ; itt!=dataVector_.end(); itt++ )
-//     COUT(0) << ' ' << (*itt).objID;
+//     COUT(0) << " " << (*itt).objID;
 //   COUT(0) << endl;
   for(it=dataVector_.begin(); it!=dataVector_.end();){
     SynchronisableHeader oldobjectheader(origdata);
     SynchronisableHeader newobjectheader(newdata);
-    if ( it->objSize == 0 )
+    if ( (*it).objSize == 0 )
     {
       ++it;
       continue;
     }
-    objectsize = oldobjectheader.getDataSize();
+    objectsize = oldobjectheader.getDataSize()+SynchronisableHeader::getSize();
     objectOffset=SynchronisableHeader::getSize(); //skip the size and the availableData variables in the objectheader
-    if ( it->objID == oldobjectheader.getObjectID() ){
+    if ( (*it).objID == oldobjectheader.getObjectID() ){
       memcpy(newdata, origdata, objectsize);
-      assert(newobjectheader.isDataAvailable()==true);
       ++it;
     }else{
       newobjectheader = oldobjectheader;
-      newobjectheader.setDataAvailable(false);
       memset(newdata+objectOffset, 0, objectsize-objectOffset);
     }
@@ -559 +586 @@
   {
     SynchronisableHeader oldobjectheader(origdata);
-    objectsize = oldobjectheader.getDataSize();
+    objectsize = oldobjectheader.getDataSize()+SynchronisableHeader::getSize();
     origdata += objectsize;
     origsize += objectsize;
@@ -571 +598 @@
 
 
-/*Gamestate *Gamestate::undiff(Gamestate *base)
-{
-  assert(this && base);assert(data_);
-  assert(header_->isDiffed());
-  assert(!header_->isCompressed() && !base->header_->isCompressed());
-  uint8_t *basep = GAMESTATE_START(base->data_);
-  uint8_t *gs = GAMESTATE_START(this->data_);
-  uint32_t of=0; // pointers offset
-  uint32_t dest_length=0;
-  dest_length=header_->getDataSize();
-  if(dest_length==0)
-    return NULL;
-  uint8_t *ndata = new uint8_t[dest_length*sizeof(uint8_t)+GamestateHeader::getSize()];
-  uint8_t *dest = ndata + GamestateHeader::getSize();
-  while(of < base->header_->getDataSize() && of < header_->getDataSize()){
-    *(dest+of)=*(basep+of)^*(gs+of); // do the xor
-    ++of;
-  }
-  if(base->header_->getDataSize()!=header_->getDataSize()){
-    uint8_t n=0;
-    if(base->header_->getDataSize() < header_->getDataSize()){
-      while(of < dest_length){
-        *(dest+of)=n^*(gs+of);
-        of++;
-      }
-    }
-  }
-  Gamestate *g = new Gamestate(ndata, getClientID());
-  assert(g->header_);
-  *(g->header_) = *header_;
-  g->header_->setDiffed( false );
-  g->flags_=flags_;
-  g->packetDirection_ = packetDirection_;
-  assert(!g->isDiffed());
-  assert(!g->isCompressed());
-  return g;
-}*/
-
 uint32_t Gamestate::calcGamestateSize(int32_t id, uint8_t mode)
 {
-  uint32_t size=0;
+  uint32_t size = 0;
+  uint32_t nrOfVariables = 0;
     // get the start of the Synchronisable list
   ObjectList<Synchronisable>::iterator it;
     // get total size of gamestate
-  for(it = ObjectList<Synchronisable>::begin(); it; ++it)
+  for(it = ObjectList<Synchronisable>::begin(); it; ++it){
     size+=it->getSize(id, mode); // size of the actual data of the synchronisable
+    nrOfVariables += it->getNrOfVariables();
+  }
+//   COUT(0) << "allocating " << nrOfVariables << " ints" << endl;
+  this->sizes_.reserve(nrOfVariables);
   return size;
 }
+
 
 } //namespace packet

code/branches/network2/src/libraries/network/packet/Gamestate.h

@@ -36 +36 @@
 #include <cstring>
 #include <list>
+#include <vector>
 
 #include "util/CRC32.h"
@@ -113 +114 @@
     inline bool isCompressed() const { return header_->isCompressed(); }
     inline int32_t getBaseID() const { return header_->getBaseID(); }
+    inline uint32_t getDataSize() const { return header_->getDataSize(); }
     Gamestate *diff(Gamestate *base);
-    Gamestate *undiff(Gamestate *base);
     Gamestate* doSelection(unsigned int clientID, unsigned int targetSize);
     bool compressData();
@@ -123 +124 @@
   private:
     void rawDiff( uint8_t* newdata, uint8_t* data, uint8_t* basedata, uint32_t datalength, uint32_t baselength);
+    inline uint32_t findObject( const SynchronisableHeader& header, uint8_t* mem, uint32_t dataLength, uint32_t startPosition = 0 );
     virtual uint32_t getSize() const;
     virtual inline bool process();
-
-  private:
     uint32_t calcGamestateSize(int32_t id, uint8_t mode=0x0);
-    std::list<obj> dataVector_;
-    GamestateHeader* header_;
+    
+    std::list<obj>          dataVector_;
+    GamestateHeader*        header_;
+    std::vector<uint32_t>   sizes_;
+    uint32_t                nrOfVariables_;
 };
 

code/branches/network2/src/libraries/network/synchronisable/Serialise.h

@@ -72 +72 @@
             return *(uint32_t*)(mem) == variable->getObjectID();
         else
-            return *(uint32_t*)(mem) == OBJECTID_UNKNOWN;
+            return variable == variable->getSynchronisable(*(uint32_t*)(mem));
     }
 }

code/branches/network2/src/libraries/network/synchronisable/Synchronisable.cc

@@ -101 +101 @@
         deletedObjects_.push(objectID_);
     }
-    // delete all Synchronisable Variables from syncList ( which are also in stringList )
-    for(std::vector<SynchronisableVariableBase*>::iterator it = syncList.begin(); it!=syncList.end(); it++)
+    // delete all Synchronisable Variables from syncList_ ( which are also in stringList_ )
+    for(std::vector<SynchronisableVariableBase*>::iterator it = syncList_.begin(); it!=syncList_.end(); it++)
       delete (*it);
-    syncList.clear();
-    stringList.clear();
+    syncList_.clear();
+    stringList_.clear();
     std::map<uint32_t, Synchronisable*>::iterator it;
     it = objectMap_.find(objectID_);
@@ -132 +132 @@
    * @return pointer to the newly created synchronisable
    */
-  Synchronisable *Synchronisable::fabricate(uint8_t*& mem, uint8_t mode)
+  Synchronisable *Synchronisable::fabricate(uint8_t*& mem, bool diffed, uint8_t mode)
   {
     SynchronisableHeader header(mem);
-
-    if(!header.isDataAvailable())
-    {
-      mem += header.getDataSize();
-      return 0;
-    }
 
     COUT(4) << "fabricating object with id: " << header.getObjectID() << std::endl;
@@ -160 +154 @@
       if (!synchronisable_creator)
       {
-        mem += header.getDataSize(); //.TODO: this suckz.... remove size from header
+        mem += header.getDataSize()+SynchronisableHeader::getSize(); //.TODO: this suckz.... remove size from header
         assert(0); // TODO: uncomment this if we have a clean objecthierarchy (with destruction of children of objects) ^^
         return 0;
@@ -245 +239 @@
    * @return true: if !doSync or if everything was successfully saved
    */
-  uint32_t Synchronisable::getData(uint8_t*& mem, int32_t id, uint8_t mode){
+  uint32_t Synchronisable::getData(uint8_t*& mem, std::vector<uint32_t>& sizes, int32_t id, uint8_t mode){
+    unsigned int test = 0;
     if(mode==0x0)
       mode=state_;
@@ -253 +248 @@
     uint32_t tempsize = 0;
 #ifndef NDEBUG
+    uint8_t* oldmem = mem;
     if (this->classID_==0)
       COUT(3) << "classid 0 " << this->getIdentifier()->getName() << std::endl;
@@ -270 +266 @@
     // end copy header
 
-
-    COUT(5) << "Synchronisable getting data from objectID_: " << objectID_ << " classID_: " << classID_ << std::endl;
+    CCOUT(5) << "getting data from objectID_: " << objectID_ << ", classID_: " << classID_ << std::endl;
+//     COUT(4) << "objectid: " << this->objectID_ << ":";
     // copy to location
-    for(i=syncList.begin(); i!=syncList.end(); ++i){
-      tempsize += (*i)->getData( mem, mode );
+    for(i=syncList_.begin(); i!=syncList_.end(); ++i){
+      uint32_t varsize = (*i)->getData( mem, mode );
+//       COUT(4) << " " << varsize;
+      tempsize += varsize;
+      sizes.push_back(varsize);
+      ++test;
       //tempsize += (*i)->getSize( mode );
     }
-
-    tempsize += SynchronisableHeader::getSize();
+//     COUT(4) << endl;
+
     header.setObjectID( this->objectID_ );
     header.setCreatorID( this->creatorID_ );
     header.setClassID( this->classID_ );
-    header.setDataAvailable( true );
     header.setDataSize( tempsize );
+    assert( tempsize == mem-oldmem-SynchronisableHeader::getSize() );
+    assert( test == this->getNrOfVariables() );
+    header.setDiffed(false);
+    tempsize += SynchronisableHeader::getSize();
 
 #ifndef NDEBUG
@@ -303 +306 @@
     if(mode==0x0)
       mode=state_;
-    std::vector<SynchronisableVariableBase *>::iterator i;
-    if(syncList.empty()){
+    if(syncList_.empty()){
       assert(0);
-      COUT(4) << "Synchronisable::updateData syncList is empty" << std::endl;
+      COUT(4) << "Synchronisable::updateData syncList_ is empty" << std::endl;
       return false;
     }
@@ -316 +318 @@
     assert(syncHeader.getCreatorID()==this->creatorID_);
     assert(syncHeader.getClassID()==this->classID_);
-    if(syncHeader.isDataAvailable()==false){
-      mem += syncHeader.getDataSize();
-      return true;
-    }
 
     mem += SynchronisableHeader::getSize();
@@ -325 +323 @@
 
     //COUT(5) << "Synchronisable: objectID_ " << syncHeader.getObjectID() << ", classID_ " << syncHeader.getClassID() << " size: " << syncHeader.getDataSize() << " synchronising data" << std::endl;
-    for(i=syncList.begin(); i!=syncList.end(); i++)
-    {
-      assert( mem <= data+syncHeader.getDataSize() ); // always make sure we don't exceed the datasize in our stream
-      (*i)->putData( mem, mode, forceCallback );
-    }
-    assert(mem == data+syncHeader.getDataSize());
+    if( !syncHeader.isDiffed() )
+    {
+      std::vector<SynchronisableVariableBase *>::iterator i;
+      for(i=syncList_.begin(); i!=syncList_.end(); i++)
+      {
+        assert( mem <= data+syncHeader.getDataSize()+SynchronisableHeader::getSize() ); // always make sure we don't exceed the datasize in our stream
+        (*i)->putData( mem, mode, forceCallback );
+      }
+    }
+    else
+    {
+      COUT(0) << "objectID: " << this->objectID_ << endl;
+      while( mem < data+syncHeader.getDataSize()+SynchronisableHeader::getSize() )
+      {
+        uint32_t varID = *(uint32_t*)mem;
+        COUT(0) << "varID: " << varID << endl;
+        if( varID == 22 )
+          COUT(6) << " blub " << endl;
+        assert( varID < syncList_.size() );
+        mem += sizeof(uint32_t);
+        syncList_[varID]->putData( mem, mode, forceCallback );
+      }
+    }
+    assert(mem == data+syncHeader.getDataSize()+SynchronisableHeader::getSize() );
     return true;
   }
@@ -341 +357 @@
   */
   uint32_t Synchronisable::getSize(int32_t id, uint8_t mode){
-    int tsize=SynchronisableHeader::getSize();
+    uint32_t tsize=SynchronisableHeader::getSize();
     if (mode==0x0)
       mode=state_;
@@ -349 +365 @@
     tsize += this->dataSize_;
     std::vector<SynchronisableVariableBase*>::iterator i;
-    for(i=stringList.begin(); i!=stringList.end(); ++i){
+    for(i=stringList_.begin(); i!=stringList_.end(); ++i){
       tsize += (*i)->getSize( mode );
     }
@@ -363 +379 @@
     if(mode==0x0)
       mode=state_;
-    return ( (this->objectMode_ & mode)!=0 && (!syncList.empty() ) );
-  }
-
-  /**
-   * This function looks at the header located in the bytestream and checks wheter objectID_ and classID_ match with the Synchronisables ones
-   * @param mem pointer to the bytestream
-   */
-  bool Synchronisable::isMyData(uint8_t* mem)
-  {
-    SynchronisableHeader header(mem);
-    assert(header.getObjectID()==this->objectID_);
-    return header.isDataAvailable();
+    return ( (this->objectMode_ & mode)!=0 && (!syncList_.empty() ) );
   }
 
@@ -397 +402 @@
     else
       sv = new SynchronisableVariable<std::string>(variable, mode, cb);
-    syncList.push_back(sv);
-    stringList.push_back(sv);
+    syncList_.push_back(sv);
+    stringList_.push_back(sv);
   }
 

code/branches/network2/src/libraries/network/synchronisable/Synchronisable.h

@@ -71 +71 @@
    * in an emulated bitset.
    * Bit 1 to 31 store the size of the Data the synchronisable consumes in the stream
-   * Bit 32 is a bool and defines whether the data is actually stored or is just filled up with 0
+   * Bit 32 is a bool and defines whether the variables are stored in diff mode
    * Byte 5 to 8: objectID_
    * Byte 9 to 12: classID_
@@ -88 +88 @@
       inline void setDataSize(uint32_t size)
         { *(uint32_t*)(data_) = (size & 0x7FFFFFFF) | (*(uint32_t*)(data_) & 0x80000000 ); }
-      inline bool isDataAvailable() const
+      inline bool isDiffed() const
         { return ( (*(uint32_t*)data_) & 0x80000000 ) == 0x80000000; }
-      inline void setDataAvailable( bool b)
+      inline void setDiffed( bool b)
         { *(uint32_t*)(data_) = (b << 31) | (*(uint32_t*)(data_) & 0x7FFFFFFF ); }
       inline uint32_t getObjectID() const
@@ -108 +108 @@
   };
 
+    /**
+   * @brief: stores information about a Synchronisable (light version)
+   *
+   * This class stores the information about a Synchronisable (objectID_, dataSize)
+   * in an emulated bitset.
+   * Bit 1 to 31 store the size of the Data the synchronisable consumes in the stream
+   * Bit 32 is a bool and defines whether the variables are stored in diff mode
+   * Byte 5 to 8: objectID_
+   */
+  class _NetworkExport SynchronisableHeaderLight{
+    private:
+      uint8_t *data_;
+    public:
+      SynchronisableHeader(uint8_t* data)
+        { data_ = data; }
+      inline static uint32_t getSize()
+        { return 16; }
+      inline uint32_t getDataSize() const
+        { return (*(uint32_t*)data_) & 0x7FFFFFFF; } //only use the first 31 bits
+      inline void setDataSize(uint32_t size)
+        { *(uint32_t*)(data_) = (size & 0x7FFFFFFF) | (*(uint32_t*)(data_) & 0x80000000 ); }
+      inline bool isDiffed() const
+        { return ( (*(uint32_t*)data_) & 0x80000000 ) == 0x80000000; }
+      inline void setDiffed( bool b)
+        { *(uint32_t*)(data_) = (b << 31) | (*(uint32_t*)(data_) & 0x7FFFFFFF ); }
+      inline uint32_t getObjectID() const
+        { return *(uint32_t*)(data_+4); }
+      inline void setObjectID(uint32_t objectID_)
+        { *(uint32_t*)(data_+4) = objectID_; }
+      inline void operator=(SynchronisableHeader& h)
+        { memcpy(data_, h.data_, getSize()); }
+  };
 
   /**
@@ -121 +153 @@
     static void setClient(bool b);
 
-    static Synchronisable *fabricate(uint8_t*& mem, uint8_t mode=0x0);
+    static Synchronisable *fabricate(uint8_t*& mem, bool diffed, uint8_t mode=0x0);
     static bool deleteObject(uint32_t objectID_);
     static Synchronisable *getSynchronisable(uint32_t objectID_);
@@ -134 +166 @@
 
     void setSyncMode(uint8_t mode);
+    
+    inline uint32_t getNrOfVariables(){ return this->syncList_.size(); }
+    inline uint32_t getVarSize( uint32_t ID )
+    { return this->syncList_[ID]->getSize(state_); }
 
   protected:
@@ -143 +179 @@
 
   private:
-    uint32_t getData(uint8_t*& men, int32_t id, uint8_t mode=0x0);
+    uint32_t getData(uint8_t*& mem, std::vector<uint32_t>& sizes, int32_t id, uint8_t mode);
     uint32_t getSize(int32_t id, uint8_t mode=0x0);
     bool updateData(uint8_t*& mem, uint8_t mode=0x0, bool forceCallback=false);
-    bool isMyData(uint8_t* mem);
     bool doSync(int32_t id, uint8_t mode=0x0);
 
@@ -156 +191 @@
     uint32_t classID_;
 
-    std::vector<SynchronisableVariableBase*> syncList;
-    std::vector<SynchronisableVariableBase*> stringList;
+    std::vector<SynchronisableVariableBase*> syncList_;
+    std::vector<SynchronisableVariableBase*> stringList_;
     uint32_t dataSize_; //size of all variables except strings
     static uint8_t state_; // detemines wheter we are server (default) or client
@@ -171 +206 @@
     if (bidirectional)
     {
-      syncList.push_back(new SynchronisableVariableBidirectional<T>(variable, mode, cb));
-      this->dataSize_ += syncList.back()->getSize(state_);
+      syncList_.push_back(new SynchronisableVariableBidirectional<T>(variable, mode, cb));
+      this->dataSize_ += syncList_.back()->getSize(state_);
     }
     else
     {
-      syncList.push_back(new SynchronisableVariable<T>(variable, mode, cb));
+      syncList_.push_back(new SynchronisableVariable<T>(variable, mode, cb));
       if ( this->state_ == mode )
-        this->dataSize_ += syncList.back()->getSize(state_);
+        this->dataSize_ += syncList_.back()->getSize(state_);
     }
   }