First draft of storage APIs (#122)

* First draft of storage APIs

* bumped pxt-core

* using fixed instances to seprate temp from permanent

* send console to storage

* sim support

* missing sim stubs

* adding storage blocks

* more sim support

* remove storage from default package

* fix rendering of ms

* render raw ms

* slize at better place

* duplicate bundled dir

* refactor limit

* simplify limit logic

libs/core/_locales/core-jsdoc-strings.json

@@ -10,10 +10,12 @@
   "MMap.getNumber": "Read a number in specified format from the buffer.",
   "MMap.ioctl": "Perform ioctl(2) on the underlaying file",
   "MMap.length": "Returns the length of a Buffer object.",
+  "MMap.lseek": "Set pointer on the underlaying file.",
   "MMap.read": "Perform read(2) on the underlaying file",
   "MMap.setNumber": "Write a number in specified format in the buffer.",
   "MMap.slice": "Read a range of bytes into a buffer.",
   "MMap.write": "Perform write(2) on the underlaying file",
+  "SeekWhence": "Mode for lseek()",
   "brick.Button": "Generic button class, for device buttons and sensors.",
   "brick.Button.isPressed": "Check if button is currently pressed or not.",
   "brick.Button.onEvent": "Do something when a button or sensor is clicked, up or down.",

libs/core/enums.d.ts

@@ -1,6 +1,17 @@
 // Auto-generated. Do not edit.
 
 
+    /**
+     * Mode for lseek()
+     */
+
+    declare const enum SeekWhence {
+    Set = 0,
+    Current = 1,
+    End = 2,
+    }
+
+
     /**
      * Drawing modes
      */

libs/core/mmap.cpp

@@ -7,6 +7,16 @@
 #include <fcntl.h>
 #include <sys/ioctl.h>
 
+/**
+* Mode for lseek()
+*/
+enum class SeekWhence {
+    Set = 0,
+    Current = 1,
+    End = 2,
+};
+
+
 namespace pxt {
 PXT_VTABLE_CTOR(MMap) {
     length = 0;
@@ -111,4 +121,10 @@ int read(MMap *mmap, Buffer data) {
     return ::read(mmap->fd, data->data, data->length);
 }
 
+/** Set pointer on the underlaying file. */
+//%
+int lseek(MMap *mmap, int offset, SeekWhence whence) {
+    return ::lseek(mmap->fd, offset, (int)whence);
+}
+
 }

libs/core/shims.d.ts

@@ -41,6 +41,10 @@ declare interface MMap {
     /** Perform read(2) on the underlaying file */
     //% shim=MMapMethods::read
     read(data: Buffer): int32;
+
+    /** Set pointer on the underlaying file. */
+    //% shim=MMapMethods::lseek
+    lseek(offset: int32, whence: SeekWhence): int32;
 }
 declare namespace control {
 

libs/ev3/ns.ts

@@ -56,4 +56,9 @@ namespace loops {
 //% color="#1E5AA8"
 namespace light {
 
+}
+
+//% color="#b0b0b0" advanced=true weight=5
+namespace storage {
+
 }

libs/storage/_locales/storage-jsdoc-strings.json

@@ -0,0 +1,26 @@
+{
+  "storage.Storage.append": "Append string data to a new or existing file.",
+  "storage.Storage.appendBuffer": "Append a buffer to a new or existing file.",
+  "storage.Storage.appendCSV": "Append a row of CSV data",
+  "storage.Storage.appendCSV|param|data": "the data to append",
+  "storage.Storage.appendCSV|param|filename": "the file name to append data, eg: \"data.txt\"",
+  "storage.Storage.appendLine": "Appends a new line of data in the file",
+  "storage.Storage.appendLine|param|data": "the data to append",
+  "storage.Storage.appendLine|param|filename": "the file name to append data, eg: \"data.txt\"",
+  "storage.Storage.append|param|data": "the data to append",
+  "storage.Storage.append|param|filename": "the file name to append data, eg: \"data.txt\"",
+  "storage.Storage.exists": "Tests if a file exists",
+  "storage.Storage.exists|param|filename": "the file name to append data, eg: \"data.txt\"",
+  "storage.Storage.limit": "Resizing the size of a file to stay under the limit",
+  "storage.Storage.limit|param|filename": "name of the file to drop",
+  "storage.Storage.limit|param|size": "maximum length",
+  "storage.Storage.overwrite": "Overwrite file with string data.",
+  "storage.Storage.overwriteWithBuffer": "Overwrite file with a buffer.",
+  "storage.Storage.overwrite|param|data": "the data to append",
+  "storage.Storage.overwrite|param|filename": "the file name to append data, eg: \"data.txt\"",
+  "storage.Storage.read": "Read contents of file as a string.",
+  "storage.Storage.readAsBuffer": "Read contents of file as a buffer.",
+  "storage.Storage.remove": "Delete a file, or do nothing if it doesn't exist.",
+  "storage.Storage.size": "Return the size of the file, or -1 if it doesn't exists.",
+  "storage.temporary": "Temporary storage in memory, deleted when the device restarts."
+}

libs/storage/_locales/storage-strings.json

@@ -0,0 +1,14 @@
+{
+  "storage.Storage.appendCSV|block": "storage %source|%filename|append CSV %data",
+  "storage.Storage.appendLine|block": "storage %source|%filename|append line %data",
+  "storage.Storage.append|block": "storage %source|%filename|append %data",
+  "storage.Storage.exists|block": "storage %source|%filename|exists",
+  "storage.Storage.limit|block": "storage %source|limit %filename|to %size|bytes",
+  "storage.Storage.overwrite|block": "storage %source|%filename|overwrite with|%data",
+  "storage.Storage.read|block": "storage %source|read %filename|as string",
+  "storage.Storage.remove|block": "storage %source|remove %filename",
+  "storage.Storage.size|block": "storage %source|%filename|size",
+  "storage.temporary|block": "temporary",
+  "storage|block": "storage",
+  "{id:category}Storage": "Storage"
+}

libs/storage/pxt.json

@@ -0,0 +1,14 @@
+{
+    "name": "storage",
+    "description": "USB Pen-drive support and flash storage",
+    "files": [
+        "storage.cpp",
+        "storage-core.ts",
+        "storage.ts",
+        "shims.d.ts"
+    ],
+    "public": true,
+    "dependencies": {
+        "core": "file:../core"
+    }
+}

libs/storage/shims.d.ts

@@ -0,0 +1,17 @@
+// Auto-generated. Do not edit.
+declare namespace storage {
+
+    /** Will be moved. */
+    //% shim=storage::__stringToBuffer
+    function __stringToBuffer(s: string): Buffer;
+
+    /** Will be moved. */
+    //% shim=storage::__bufferToString
+    function __bufferToString(s: Buffer): string;
+
+    /** Create named directory. */
+    //% shim=storage::__mkdir
+    function __mkdir(filename: string): void;
+}
+
+// Auto-generated. Do not edit. Really.

libs/storage/storage-core.ts

@@ -0,0 +1,178 @@
+namespace storage {
+    //% shim=storage::__unlink
+    function __unlink(filename: string): void { }
+    //% shim=storage::__truncate
+    function __truncate(filename: string): void { }
+
+    //% fixedInstances
+    export class Storage {
+        constructor() { }
+
+        protected mapFilename(filename: string) {
+            return filename;
+        }
+
+        private getFile(filename: string): MMap {
+            filename = this.mapFilename(filename)
+            let r = control.mmap(filename, 0, 0)
+            if (!r) {
+                __mkdir(this.dirname(filename))
+                __truncate(filename)
+                r = control.mmap(filename, 0, 0)
+            }
+            if (!r)
+                control.panic(906)
+            return r
+        }
+
+        dirname(filename: string) {
+            let last = 0
+            for (let i = 0; i < filename.length; ++i)
+                if (filename[i] == "/")
+                    last = i
+            return filename.substr(0, last)
+        }
+
+        /**
+         *  Append string data to a new or existing file. 
+         * @param filename the file name to append data, eg: "data.txt"
+         * @param data the data to append
+         */
+        //% blockId=storageAppend block="storage %source|%filename|append %data"
+        append(filename: string, data: string): void {
+            this.appendBuffer(filename, __stringToBuffer(data))
+        }
+
+        /**
+         * Appends a new line of data in the file
+         * @param filename the file name to append data, eg: "data.txt"
+         * @param data the data to append
+         */
+        //% blockId=storageAppendLine block="storage %source|%filename|append line %data"
+        appendLine(filename: string, data: string): void {
+            this.append(filename, data + "\r\n");
+        }
+
+        /** Append a buffer to a new or existing file. */
+        appendBuffer(filename: string, data: Buffer): void {
+            let f = this.getFile(filename);
+            f.lseek(0, SeekWhence.End)
+            f.write(data)
+        }
+
+        /**
+         * Append a row of CSV data
+         * @param filename the file name to append data, eg: "data.txt"
+         * @param data the data to append
+         */
+        //% blockId=storageAppendCSV block="storage %source|%filename|append CSV %data"
+        appendCSV(filename: string, data: number[]) {
+            let s = ""
+            for (const d of data) {
+                if (s) s += "\t"
+                s = s + d;
+            }
+            s += "\r\n"
+            this.append(filename, s)
+        }
+
+        /** Overwrite file with string data.
+         * @param filename the file name to append data, eg: "data.txt"
+         * @param data the data to append
+         */
+        //% blockId=storageOverwrite block="storage %source|%filename|overwrite with|%data"
+        overwrite(filename: string, data: string): void {
+            this.overwriteWithBuffer(filename, __stringToBuffer(data))
+        }
+
+        /** Overwrite file with a buffer. */
+        overwriteWithBuffer(filename: string, data: Buffer): void {
+            __truncate(this.mapFilename(filename))
+            this.appendBuffer(filename, data)
+        }
+
+        /** Tests if a file exists
+         * @param filename the file name to append data, eg: "data.txt"
+         */
+        //% blockId=storageExists block="storage %source|%filename|exists"
+        exists(filename: string): boolean {
+            return !!control.mmap(this.mapFilename(filename), 0, 0);
+        }
+
+        /** Delete a file, or do nothing if it doesn't exist. */
+        //% blockId=storageRemove block="storage %source|remove %filename"
+        remove(filename: string): void {
+            __unlink(this.mapFilename(filename))
+        }
+
+        /** Return the size of the file, or -1 if it doesn't exists. */
+        //% blockId=storageSize block="storage %source|%filename|size"
+        size(filename: string): int32 {
+            let f = control.mmap(this.mapFilename(filename), 0, 0)
+            if (!f) return -1;
+            return f.lseek(0, SeekWhence.End)
+        }
+
+        /** Read contents of file as a string. */
+        //% blockId=storageRead block="storage %source|read %filename|as string"
+        read(filename: string): string {
+            return __bufferToString(this.readAsBuffer(filename))
+        }
+
+        /** Read contents of file as a buffer. */
+        //%
+        readAsBuffer(filename: string): Buffer {
+            let f = this.getFile(filename)
+            let sz = f.lseek(0, SeekWhence.End)
+            let b = output.createBuffer(sz)
+            f.lseek(0, SeekWhence.Set);
+            f.read(b)
+            return b
+        }
+
+        /**
+         * Resizing the size of a file to stay under the limit
+         * @param filename name of the file to drop
+         * @param size maximum length
+         */
+        //% blockId=storageLimit block="storage %source|limit %filename|to %size|bytes"
+        limit(filename: string, size: number) {
+            if (!this.exists(filename) || size < 0) return;
+
+            const sz = storage.temporary.size(filename);
+            if (sz > size) {
+                let buf = storage.temporary.readAsBuffer(filename)
+                buf = buf.slice(buf.length / 2);
+                storage.temporary.overwriteWithBuffer(filename, buf);
+            }
+        }
+    }
+
+    class TemporaryStorage extends Storage {
+        constructor() {
+            super();
+        }
+
+        protected mapFilename(filename: string) {
+            if (filename[0] == '/') filename = filename.substr(1);
+            return '/tmp/logs/' + filename;
+        }
+    }
+
+    /**
+     * Temporary storage in memory, deleted when the device restarts.
+     */
+    //% whenUsed fixedInstance block="temporary"
+    export const temporary: Storage = new TemporaryStorage();
+
+    class PermanentStorage extends Storage {
+        constructor() {
+            super()
+        }
+
+        protected mapFilename(filename: string) {
+            if (filename[0] == '/') return filename;
+            return '/' + filename;
+        }
+    }
+}

libs/storage/storage.cpp

@@ -0,0 +1,44 @@
+#include "pxt.h"
+
+#include <sys/types.h>
+#include <unistd.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+namespace storage {
+
+/** Will be moved. */
+//%
+Buffer __stringToBuffer(String s) {
+    return mkBuffer((uint8_t *)s->data, s->length);
+}
+
+/** Will be moved. */
+//%
+String __bufferToString(Buffer s) {
+    return mkString((char*)s->data, s->length);
+}
+
+//%
+void __init() {
+    // do nothing
+}
+
+//%
+void __unlink(String filename) {
+    ::unlink(filename->data);
+}
+
+//%
+void __truncate(String filename) {
+    int fd = open(filename->data, O_CREAT | O_TRUNC | O_WRONLY, 0777);
+    close(fd);
+}
+
+/** Create named directory. */
+//%
+void __mkdir(String filename) {
+    ::mkdir(filename->data, 0777);
+}
+
+} // namespace storage

libs/storage/storage.ts

@@ -0,0 +1,11 @@
+namespace storage {
+    // automatically send console output to temp storage
+    storage.temporary.remove("console.txt");
+    console.addListener(function(line) {
+        const fn = "console.txt";
+        const mxs = 65536;
+        const t = control.millis();
+        storage.temporary.appendLine(fn, `${t}> ${line}`);
+        storage.temporary.limit(fn, 65536);
+    })    
+}

pxtarget.json

@@ -16,6 +16,7 @@
         "libs/infrared-sensor",
         "libs/gyro-sensor",
         "libs/chassis",
+        "libs/storage",
         "libs/ev3",
         "libs/tests",
         "libs/behaviors"

sim/state/control.ts

@@ -8,6 +8,7 @@ namespace pxsim.MMapMethods {
         read?: (d: Buffer) => number;
         write?: (d: Buffer) => number;
         ioctl?: (id: number, d: Buffer) => number;
+        lseek?: (offset: number, whence: number) => number;
     }
 
     import BM = pxsim.BufferMethods
@@ -23,6 +24,7 @@ namespace pxsim.MMapMethods {
             if (!impl.read) impl.read = () => 0
             if (!impl.write) impl.write = () => 0
             if (!impl.ioctl) impl.ioctl = () => -1
+            if (!impl.lseek) impl.lseek = (offset, whence) => -1
         }
         destroy() {
         }
@@ -68,6 +70,10 @@ namespace pxsim.MMapMethods {
     export function read(m: MMap, data: Buffer): number {
         return m.impl.read(data)
     }
+
+    export function lseek(m: MMap, offset: number, whence: number): number {
+        return m.impl.lseek(offset, whence);
+    }
 }
 
 namespace pxsim.control {

sim/state/storage.ts

@@ -0,0 +1,23 @@
+namespace pxsim.storage {
+    export function __stringToBuffer(s: string): RefBuffer {
+        // TODO
+        return new RefBuffer(new Uint8Array([]));
+    }
+
+    export function __bufferToString(b: RefBuffer): string {
+        // TODO
+        return "";
+    }
+
+    export function __mkdir(fn: string) {
+        // TODO
+    }
+
+    export function __unlink(filename: string): void { 
+        // TODO
+    }
+
+    export function __truncate(filename: string): void { 
+        // TODO
+    }
+}