Array reimplementation as single segment with support for missing values

libs/core/core.cpp

@@ -155,6 +155,8 @@ namespace Array_ {
     //%
     void push(RefCollection *c, uint32_t x) { c->push(x); }
     //%
+    uint32_t pop(RefCollection *c) { return c->pop(); }    
+    //%
     uint32_t getAt(RefCollection *c, int x) { return c->getAt(x); }
     //%
     void removeAt(RefCollection *c, int x) { c->removeAt(x); }

libs/core/pxt.cpp

@@ -141,63 +141,290 @@ namespace pxt {
       printf("RefRecord %p r=%d size=%d bytes\n", r, r->refcnt, r->getVTable()->numbytes);
     }
 
-    void RefCollection::push(uint32_t x) {
+    uint32_t Segment::get(uint32_t i)
-      if (isRef()) incr(x);
+    {
-      data.push_back(x);
+#ifdef DEBUG_BUILD
+      printf("In Segment::get index:%u\n", i);
+      this->print();
+#endif            
+      
+      if (i < length)
+      {
+        if (data[i] != Segment::MissingValue)
+        {
+          return data[i];          
+        }
+        error(ERR_MISSING_VALUE);
+        return 0;
+      }
+      error(ERR_OUT_OF_BOUNDS);
+      return 0;
     }
 
-    uint32_t RefCollection::getAt(int x) {
+    void Segment::set(uint32_t i, uint32_t value) 
-      if (in_range(x)) {
+    {
-        uint32_t tmp = data.at(x);
+        if (i < size)
-        if (isRef()) incr(tmp);
+        {
+          data[i] = value;
+        }
+        else if (i < Segment::MaxSize)
+        {
+          growByMin(i + 1);
+          data[i] = value;
+        }
+        if (length <= i)
+        {
+           length = i + 1; 
+        }        
+
+#ifdef DEBUG_BUILD
+        printf("In Segment::set\n");
+        this->print();
+#endif            
+        
+        return;
+    }      
+
+    uint16_t Segment::growthFactor(uint16_t size)
+    {
+      if (size == 0)
+      {
+        return 4;
+      }
+      if (size < 64)
+      {
+        return size * 2; // Double
+      }
+      if (size < 512)
+      {
+        return size * 5/3; //Grow by 1.66 rate
+      }
+      return size + 256; //Grow by constant rate
+    }
+
+    void Segment::growByMin(uint16_t minSize)
+    {
+      growBy(max(minSize, growthFactor(size)));
+    }
+
+    void Segment::growBy(uint16_t newSize) 
+    {
+#ifdef DEBUG_BUILD
+         printf("growBy: %d\n", newSize);
+         this->print();
+#endif      
+      if (size < newSize)
+      {
+         //this will throw if unable to allocate
+         uint32_t *tmp = (uint32_t*)(::operator new(newSize * sizeof(uint32_t)));
+
+         //Copy existing data
+         if (size)
+         {
+           memcpy(tmp, data, size * sizeof(uint32_t));
+         }
+
+         //fill the rest with missing values;
+         for(uint16_t i = size; i < newSize; i++)
+         {
+           tmp[i] = Segment::MissingValue;
+         }
+
+         //free older segment;
+         ::operator delete(data); 
+
+         data = tmp; 
+         size = newSize;
+
+#ifdef DEBUG_BUILD
+         printf("growBy - after reallocation\n");
+         this->print();
+#endif         
+         
+      }
+      //else { no shrinking yet; }
+      return;
+    }
+
+    void Segment::push(uint32_t value) 
+    { 
+      this->set(length, value);
+    }
+    
+    uint32_t Segment::pop() 
+    {
+#ifdef DEBUG_BUILD
+      printf("In Segment::pop\n");
+      this->print();
+#endif            
+      
+      if (length > 0)
+      {
+        uint32_t value = data[length];
+        data[length] = Segment::MissingValue;
+        --length;
+        return value;
+      }
+      error(ERR_OUT_OF_BOUNDS);      
+      return 0;
+    }
+
+    void Segment::remove(uint32_t i) 
+    {
+#ifdef DEBUG_BUILD
+      printf("In Segment::remove\n");
+      this->print();
+#endif                  
+      if (i < length)
+      {
+        data[i] = Segment::MissingValue;
+      }
+      return;
+    }
+
+    void Segment::print()
+    {
+      printf("Segment: %x, length: %u, size: %u\n", data, (uint)length, (uint)size);
+      for(uint i = 0; i < size; i++)
+      {
+        printf("%d ",(uint)data[i]);
+      }
+      printf("\n");
+    }
+
+    bool Segment::isValidIndex(uint32_t i)
+    {
+      if (i > length || data[i] == Segment::MissingValue)
+      {
+        return false;
+      }
+      return true;
+    }
+
+    bool Segment::getNextValidIndex(uint32_t i, uint32_t *result)
+    {
+      while (i < length)
+      {
+        if (data[i] != Segment::MissingValue)
+        {
+           *result = i;
+
+#ifdef DEBUG_BUILD
+           printf("In Segment::getNextValidIndex result=%u\n",i);
+           this->print();
+#endif                  
+           return true;
+        }
+        i++;
+      }
+      return false;
+    }
+
+    void Segment::destroy()
+    {
+#ifdef DEBUG_BUILD
+      printf("In Segment::destroy\n");
+      this->print();
+#endif          
+      length = size = 0;
+      ::operator delete(data);
+      data = nullptr;
+    }
+
+    void RefCollection::push(uint32_t x) 
+    {
+      if (isRef()) incr(x);
+      head.push(x);
+    }
+
+    uint32_t RefCollection::pop() 
+    {
+      uint32_t ret = head.pop();
+      if (isRef())
+      {
+        incr(ret);  
+      } 
+      return ret;
+    }
+
+    uint32_t RefCollection::getAt(int i) 
+    {
+      if (head.isValidIndex(i)) 
+      {
+        uint32_t tmp = head.get(i);
+        if (isRef())
+        {
+          incr(tmp);
+        }
         return tmp;
       }
-      else {
+      else 
+      {
         error(ERR_OUT_OF_BOUNDS);
         return 0;
       }
     }
 
-    void RefCollection::removeAt(int x) {
+    void RefCollection::removeAt(int i) 
-      if (!in_range(x))
+    {
+      if (!head.isValidIndex((uint32_t)i))
+      {
         return;
-
-      if (isRef()) decr(data.at(x));
-      data.erase(data.begin()+x);
-    }
-
-    void RefCollection::setAt(int x, uint32_t y) {
-      if (!in_range(x))
-        return;
-
-      if (isRef()) {
-        decr(data.at(x));
-        incr(y);
       }
-      data.at(x) = y;
+      if (isRef())
+      {
+        decr(head.get(i));
+      } 
+      head.remove(i);
     }
 
-    int RefCollection::indexOf(uint32_t x, int start) {
+    void RefCollection::setAt(int i, uint32_t value) 
-      if (!in_range(start))
+    {
-        return -1;
+      if (isRef()) 
-
+      {
-      if (isString()) {
+        if (head.isValidIndex((uint32_t)i))
-        StringData *xx = (StringData*)x;
+        {
-        for (uint32_t i = start; i < data.size(); ++i) {
+          decr(head.get(i));
-          StringData *ee = (StringData*)data.at(i);
-          if (xx->len == ee->len && memcmp(xx->data, ee->data, xx->len) == 0)
-            return (int)i;
         }
-      } else {
+        incr(value);
-        for (uint32_t i = start; i < data.size(); ++i)
+      }
-          if (data.at(i) == x)
+      head.set(i, value);
+    }
+
+    int RefCollection::indexOf(uint32_t x, int start) 
+    {
+      if (isString()) 
+      {
+        StringData *xx = (StringData*)x;
+        uint32_t i = start;
+        while(head.getNextValidIndex(start, &i))
+        {
+          StringData *ee = (StringData*)head.get(i);
+          if (xx->len == ee->len && memcmp(xx->data, ee->data, xx->len) == 0)
+          {
             return (int)i;
+          }
+          start = i;
+        }
+      } 
+      else 
+      {
+        uint32_t i = start;
+        while(head.getNextValidIndex(start, &i))
+        {
+          if (head.get(i) == x)
+          {
+            return (int)i;
+          }
+          start = i;
+        }
       }
 
       return -1;
     }
 
-    int RefCollection::removeElement(uint32_t x) {
+    int RefCollection::removeElement(uint32_t x) 
+    {
       int idx = indexOf(x, 0);
       if (idx >= 0) {
         removeAt(idx);
@@ -239,17 +466,23 @@ namespace pxt {
     void RefCollection::destroy()
     {
       if (this->isRef())
-        for (uint32_t i = 0; i < this->data.size(); ++i) {
+      {
-          decr(this->data[i]);
+        uint32_t start = 0;
-          this->data[i] = 0;
+        uint32_t i = 0;
+        while(head.getNextValidIndex(start, &i))
+        {
+          decr(this->head.get(i));
+          start = i;
         }
-      this->data.resize(0);
+      }
+      this->head.destroy();
       delete this;
     }
 
     void RefCollection::print()
     {
-      printf("RefCollection %p r=%d flags=%d size=%d [%p, ...]\n", this, refcnt, getFlags(), data.size(), data.size() > 0 ? data[0] : 0);
+      printf("RefCollection %p r=%d flags=%d size=%d\n", this, refcnt, getFlags(), head.getLength());
+      head.print();
     }
 
     PXT_VTABLE_CTOR(RefAction) {}

libs/core/pxt.h

@@ -1,7 +1,7 @@
 #ifndef __PXT_H
 #define __PXT_H
 
-// #define DEBUG_MEMLEAKS 1
+//#define DEBUG_MEMLEAKS 1
 
 #pragma GCC diagnostic ignored "-Wunused-parameter"
 
@@ -38,6 +38,7 @@ namespace pxt {
     ERR_OUT_OF_BOUNDS = 8,
     ERR_REF_DELETED = 7,
     ERR_SIZE = 9,
+    ERR_MISSING_VALUE = 10,
   } ERROR;
 
   extern const uint32_t functionsAndBytecode[];
@@ -167,11 +168,50 @@ namespace pxt {
     }
   };
 
+  class Segment {
+  private:    
+      uint32_t* data;
+      uint16_t length;
+      uint16_t size;
+
+      static const uint16_t MaxSize = 0xFFFF;
+      static const uint32_t MissingValue = 0x80000000;
+
+      static uint16_t growthFactor(uint16_t size);      
+      void growByMin(uint16_t minSize);
+      void growBy(uint16_t newSize);
+
+  public:
+      Segment() : data (nullptr), length(0), size(0) {};
+
+      uint32_t get(uint32_t i);
+      void set(uint32_t i, uint32_t value);      
+
+      uint32_t getLength() { return length;};
+
+      void push(uint32_t value);
+      uint32_t pop();
+
+      void remove(uint32_t i);
+
+      //Returns true if there is a valid index greater than or equal to 'i', returns false otherwise
+      //If 'i' is valid returns it in 'result', if not tries to find the next valid
+      //index < length which is valid.
+      bool getNextValidIndex(uint32_t i, uint32_t *result);
+      bool isValidIndex(uint32_t i);
+
+      void destroy();
+
+      void print();
+  };
+
   // A ref-counted collection of either primitive or ref-counted objects (String, Image,
   // user-defined record, another collection)
   class RefCollection
     : public RefObject
   {
+  private:
+    Segment head;
   public:
     // 1 - collection of refs (need decr)
     // 2 - collection of strings (in fact we always have 3, never 2 alone)
@@ -179,20 +219,14 @@ namespace pxt {
     inline bool isRef() { return getFlags() & 1; }
     inline bool isString() { return getFlags() & 2; }
 
-    std::vector<uint32_t> data;
-
     RefCollection(uint16_t f);
 
-    inline bool in_range(int x) {
-      return (0 <= x && x < (int)data.size());
-    }
-
-    inline int length() { return data.size(); }
-
     void destroy();
     void print();
 
+    uint32_t length() { return head.getLength();}
     void push(uint32_t x);
+    uint32_t pop();
     uint32_t getAt(int x);
     void removeAt(int x);
     void setAt(int x, uint32_t y);