Sqlite: Update Sqlite from 3.31.1 to 3.34 and adapt carray

You can now pass everything you can convert to a span directly and bind it
with a carray instead of using the pointer interface. This is working for
int, long long, double and null terminated C strings.

Change-Id: I274c218e2dec0f11e68576545bb78601f85462bd
Reviewed-by: Alessandro Portale <alessandro.portale@qt.io>
Reviewed-by: Thomas Hartmann <thomas.hartmann@qt.io>

src/libs/3rdparty/sqlite/carray.c

@@ -57,21 +57,32 @@ SQLITE_EXTENSION_INIT1
 #include <assert.h>
 #include <string.h>
 
+/* Allowed values for the mFlags parameter to sqlite3_carray_bind().
+** Must exactly match the definitions in carray.h.
+*/
+#define CARRAY_INT32     0      /* Data is 32-bit signed integers */
+#define CARRAY_INT64     1      /* Data is 64-bit signed integers */
+#define CARRAY_DOUBLE    2      /* Data is doubles */
+#define CARRAY_TEXT      3      /* Data is char* */
+
 #ifndef SQLITE_OMIT_VIRTUALTABLE
 
 /*
-** Allowed datatypes
+** Names of allowed datatypes
-*/
-#define CARRAY_INT32    0
-#define CARRAY_INT64    1
-#define CARRAY_DOUBLE   2
-#define CARRAY_TEXT     3
-
-/*
-** Names of types
 */
 static const char *azType[] = { "int32", "int64", "double", "char*" };
 
+/*
+** Structure used to hold the sqlite3_carray_bind() information
+*/
+typedef struct carray_bind carray_bind;
+struct carray_bind {
+  void *aData;                /* The data */
+  int nData;                  /* Number of elements */
+  int mFlags;                 /* Control flags */
+  void (*xDel)(void*);        /* Destructor for aData */
+};
+
 
 /* carray_cursor is a subclass of sqlite3_vtab_cursor which will
 ** serve as the underlying representation of a cursor that scans
@@ -136,16 +147,13 @@ static int carrayDisconnect(sqlite3_vtab *pVtab){
 /*
 ** Constructor for a new carray_cursor object.
 */
-static int carrayOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor)
+static int carrayOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
-{
+  carray_cursor *pCur;
-    carray_cursor *pCur;
+  pCur = sqlite3_malloc( sizeof(*pCur) );
-
+  if( pCur==0 ) return SQLITE_NOMEM;
-    pCur = sqlite3_malloc(sizeof(*pCur));
+  memset(pCur, 0, sizeof(*pCur));
-    if (pCur == 0)
+  *ppCursor = &pCur->base;
-        return SQLITE_NOMEM;
+  return SQLITE_OK;
-    memset(pCur, 0, sizeof(*pCur));
-    *ppCursor = &pCur->base;
-    return SQLITE_OK;
 }
 
 /*
@@ -242,28 +250,39 @@ static int carrayFilter(
   int argc, sqlite3_value **argv
 ){
   carray_cursor *pCur = (carray_cursor *)pVtabCursor;
-  if( idxNum ){
+  pCur->pPtr = 0;
-    pCur->pPtr = sqlite3_value_pointer(argv[0], "carray");
+  pCur->iCnt = 0;
-    pCur->iCnt = pCur->pPtr ? sqlite3_value_int64(argv[1]) : 0;
+  switch( idxNum ){
-    if( idxNum<3 ){
+    case 1: {
-      pCur->eType = CARRAY_INT32;
+      carray_bind *pBind = sqlite3_value_pointer(argv[0], "carray-bind");
-    }else{
+      if( pBind==0 ) break;
-      unsigned char i;
+      pCur->pPtr = pBind->aData;
-      const char *zType = (const char*)sqlite3_value_text(argv[2]);
+      pCur->iCnt = pBind->nData;
-      for(i=0; i<sizeof(azType)/sizeof(azType[0]); i++){
+      pCur->eType = pBind->mFlags & 0x03;
-        if( sqlite3_stricmp(zType, azType[i])==0 ) break;
+      break;
-      }
+    }
-      if( i>=sizeof(azType)/sizeof(azType[0]) ){
+    case 2:
-        pVtabCursor->pVtab->zErrMsg = sqlite3_mprintf(
+    case 3: {
-          "unknown datatype: %Q", zType);
+      pCur->pPtr = sqlite3_value_pointer(argv[0], "carray");
-        return SQLITE_ERROR;
+      pCur->iCnt = pCur->pPtr ? sqlite3_value_int64(argv[1]) : 0;
-      }else{
+      if( idxNum<3 ){
-        pCur->eType = i;
+        pCur->eType = CARRAY_INT32;
-      }
+      }else{
+        unsigned char i;
+        const char *zType = (const char*)sqlite3_value_text(argv[2]);
+        for(i=0; i<sizeof(azType)/sizeof(azType[0]); i++){
+          if( sqlite3_stricmp(zType, azType[i])==0 ) break;
+        }
+        if( i>=sizeof(azType)/sizeof(azType[0]) ){
+          pVtabCursor->pVtab->zErrMsg = sqlite3_mprintf(
+            "unknown datatype: %Q", zType);
+          return SQLITE_ERROR;
+        }else{
+          pCur->eType = i;
+        }
+      }
+      break;
     }
-  }else{
-    pCur->pPtr = 0;
-    pCur->iCnt = 0;
   }
   pCur->iRowid = 1;
   return SQLITE_OK;
@@ -278,9 +297,16 @@ static int carrayFilter(
 ** In this implementation idxNum is used to represent the
 ** query plan.  idxStr is unused.
 **
-** idxNum is 2 if the pointer= and count= constraints exist,
+** idxNum is:
-** 3 if the ctype= constraint also exists, and is 0 otherwise.
+**
-** If idxNum is 0, then carray becomes an empty table.
+**    1    If only the pointer= constraint exists.  In this case, the
+**         parameter must be bound using sqlite3_carray_bind().
+**
+**    2    if the pointer= and count= constraints exist.
+**
+**    3    if the ctype= constraint also exists.
+**
+** idxNum is 0 otherwise and carray becomes an empty table.
 */
 static int carrayBestIndex(
   sqlite3_vtab *tab,
@@ -308,18 +334,21 @@ static int carrayBestIndex(
         break;
     }
   }
-  if( ptrIdx>=0 && cntIdx>=0 ){
+  if( ptrIdx>=0 ){
     pIdxInfo->aConstraintUsage[ptrIdx].argvIndex = 1;
     pIdxInfo->aConstraintUsage[ptrIdx].omit = 1;
-    pIdxInfo->aConstraintUsage[cntIdx].argvIndex = 2;
-    pIdxInfo->aConstraintUsage[cntIdx].omit = 1;
     pIdxInfo->estimatedCost = (double)1;
     pIdxInfo->estimatedRows = 100;
-    pIdxInfo->idxNum = 2;
+    pIdxInfo->idxNum = 1;
-    if( ctypeIdx>=0 ){
+    if( cntIdx>=0 ){
-      pIdxInfo->aConstraintUsage[ctypeIdx].argvIndex = 3;
+      pIdxInfo->aConstraintUsage[cntIdx].argvIndex = 2;
-      pIdxInfo->aConstraintUsage[ctypeIdx].omit = 1;
+      pIdxInfo->aConstraintUsage[cntIdx].omit = 1;
-      pIdxInfo->idxNum = 3;
+      pIdxInfo->idxNum = 2;
+      if( ctypeIdx>=0 ){
+        pIdxInfo->aConstraintUsage[ctypeIdx].argvIndex = 3;
+        pIdxInfo->aConstraintUsage[ctypeIdx].omit = 1;
+        pIdxInfo->idxNum = 3;
+      }
     }
   }else{
     pIdxInfo->estimatedCost = (double)2147483647;
@@ -356,6 +385,89 @@ static sqlite3_module carrayModule = {
   0,                         /* xRename */
 };
 
+/*
+** Destructor for the carray_bind object
+*/
+static void carrayBindDel(void *pPtr){
+  carray_bind *p = (carray_bind*)pPtr;
+  if( p->xDel!=SQLITE_STATIC ){
+     p->xDel(p->aData);
+  }
+  sqlite3_free(p);
+}
+
+/*
+** Invoke this interface in order to bind to the single-argument
+** version of CARRAY().
+*/
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_carray_bind(
+  sqlite3_stmt *pStmt,
+  int idx,
+  void *aData,
+  int nData,
+  int mFlags,
+  void (*xDestroy)(void*)
+){
+  carray_bind *pNew;
+  int i;
+  pNew = sqlite3_malloc64(sizeof(*pNew));
+  if( pNew==0 ){
+    if( xDestroy!=SQLITE_STATIC && xDestroy!=SQLITE_TRANSIENT ){
+      xDestroy(aData);
+    }
+    return SQLITE_NOMEM;
+  }
+  pNew->nData = nData;
+  pNew->mFlags = mFlags;
+  if( xDestroy==SQLITE_TRANSIENT ){
+    sqlite3_int64 sz = nData;
+    switch( mFlags & 0x03 ){
+      case CARRAY_INT32:   sz *= 4;              break;
+      case CARRAY_INT64:   sz *= 8;              break;
+      case CARRAY_DOUBLE:  sz *= 8;              break;
+      case CARRAY_TEXT:    sz *= sizeof(char*);  break;
+    }
+    if( (mFlags & 0x03)==CARRAY_TEXT ){
+      for(i=0; i<nData; i++){
+        const char *z = ((char**)aData)[i];
+        if( z ) sz += strlen(z) + 1;
+      }
+    } 
+    pNew->aData = sqlite3_malloc64( sz );
+    if( pNew->aData==0 ){
+      sqlite3_free(pNew);
+      return SQLITE_NOMEM;
+    }
+    if( (mFlags & 0x03)==CARRAY_TEXT ){
+      char **az = (char**)pNew->aData;
+      char *z = (char*)&az[nData];
+      for(i=0; i<nData; i++){
+        const char *zData = ((char**)aData)[i];
+        sqlite3_int64 n;
+        if( zData==0 ){
+          az[i] = 0;
+          continue;
+        }
+        az[i] = z;
+        n = strlen(zData);
+        memcpy(z, zData, n+1);
+        z += n+1;
+      }
+    }else{
+      memcpy(pNew->aData, aData, sz*nData);
+    }
+    pNew->xDel = sqlite3_free;
+  }else{
+    pNew->aData = aData;
+    pNew->xDel = xDestroy;
+  }
+  return sqlite3_bind_pointer(pStmt, idx, pNew, "carray-bind", carrayBindDel);
+}
+
+
 /*
 ** For testing purpose in the TCL test harness, we need a method for
 ** setting the pointer value.  The inttoptr(X) SQL function accomplishes
@@ -386,16 +498,22 @@ static void inttoptrFunc(
 
 #endif /* SQLITE_OMIT_VIRTUALTABLE */
 
-int sqlite3_carray_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi)
+#ifdef _WIN32
-{
+__declspec(dllexport)
-    int rc = SQLITE_OK;
+#endif
-    SQLITE_EXTENSION_INIT2(pApi);
+int sqlite3_carray_init(
+  sqlite3 *db, 
+  char **pzErrMsg, 
+  const sqlite3_api_routines *pApi
+){
+  int rc = SQLITE_OK;
+  SQLITE_EXTENSION_INIT2(pApi);
 #ifndef SQLITE_OMIT_VIRTUALTABLE
   rc = sqlite3_create_module(db, "carray", &carrayModule, 0);
-
 #ifdef SQLITE_TEST
-  if (rc == SQLITE_OK) {
+  if( rc==SQLITE_OK ){
-      rc = sqlite3_create_function(db, "inttoptr", 1, SQLITE_UTF8, 0, inttoptrFunc, 0, 0);
+    rc = sqlite3_create_function(db, "inttoptr", 1, SQLITE_UTF8, 0,
+                                 inttoptrFunc, 0, 0);
   }
 #endif /* SQLITE_TEST */
 #endif /* SQLITE_OMIT_VIRTUALTABLE */

src/libs/3rdparty/sqlite/sqlite3ext.h

@@ -330,6 +330,13 @@ struct sqlite3_api_routines {
   const char *(*filename_database)(const char*);
   const char *(*filename_journal)(const char*);
   const char *(*filename_wal)(const char*);
+  /* Version 3.32.0 and later */
+  char *(*create_filename)(const char*,const char*,const char*,
+                           int,const char**);
+  void (*free_filename)(char*);
+  sqlite3_file *(*database_file_object)(const char*);
+  /* Version 3.34.0 and later */
+  int (*txn_state)(sqlite3*,const char*);
 };
 
 /*
@@ -630,6 +637,12 @@ typedef int (*sqlite3_loadext_entry)(
 #define sqlite3_filename_database      sqlite3_api->filename_database
 #define sqlite3_filename_journal       sqlite3_api->filename_journal
 #define sqlite3_filename_wal           sqlite3_api->filename_wal
+/* Version 3.32.0 and later */
+#define sqlite3_create_filename        sqlite3_api->create_filename
+#define sqlite3_free_filename          sqlite3_api->free_filename
+#define sqlite3_database_file_object   sqlite3_api->database_file_object
+/* Version 3.34.0 and later */
+#define sqlite3_txn_state              sqlite3_api->txn_state
 #endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */
 
 #if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION)

src/libs/sqlite/sqlitebasestatement.cpp

@@ -38,6 +38,14 @@
 #  pragma GCC diagnostic ignored "-Wignored-qualifiers"
 #endif
 
+#define CARRAY_INT32 0  /* Data is 32-bit signed integers */
+#define CARRAY_INT64 1  /* Data is 64-bit signed integers */
+#define CARRAY_DOUBLE 2 /* Data is doubles */
+#define CARRAY_TEXT 3   /* Data is char* */
+
+extern "C" int sqlite3_carray_bind(
+    sqlite3_stmt *pStmt, int idx, void *aData, int nData, int mFlags, void (*xDestroy)(void *));
+
 namespace Sqlite {
 
 BaseStatement::BaseStatement(Utils::SmallStringView sqlStatement, Database &database)
@@ -180,6 +188,54 @@ void BaseStatement::bind(int index, void *pointer)
         checkForBindingError(resultCode);
 }
 
+void BaseStatement::bind(int index, Utils::span<int> values)
+{
+    int resultCode = sqlite3_carray_bind(m_compiledStatement.get(),
+                                         index,
+                                         values.data(),
+                                         static_cast<int>(values.size()),
+                                         CARRAY_INT32,
+                                         SQLITE_STATIC);
+    if (resultCode != SQLITE_OK)
+        checkForBindingError(resultCode);
+}
+
+void BaseStatement::bind(int index, Utils::span<long long> values)
+{
+    int resultCode = sqlite3_carray_bind(m_compiledStatement.get(),
+                                         index,
+                                         values.data(),
+                                         static_cast<int>(values.size()),
+                                         CARRAY_INT64,
+                                         SQLITE_STATIC);
+    if (resultCode != SQLITE_OK)
+        checkForBindingError(resultCode);
+}
+
+void BaseStatement::bind(int index, Utils::span<double> values)
+{
+    int resultCode = sqlite3_carray_bind(m_compiledStatement.get(),
+                                         index,
+                                         values.data(),
+                                         static_cast<int>(values.size()),
+                                         CARRAY_DOUBLE,
+                                         SQLITE_STATIC);
+    if (resultCode != SQLITE_OK)
+        checkForBindingError(resultCode);
+}
+
+void BaseStatement::bind(int index, Utils::span<const char *> values)
+{
+    int resultCode = sqlite3_carray_bind(m_compiledStatement.get(),
+                                         index,
+                                         values.data(),
+                                         static_cast<int>(values.size()),
+                                         CARRAY_TEXT,
+                                         SQLITE_STATIC);
+    if (resultCode != SQLITE_OK)
+        checkForBindingError(resultCode);
+}
+
 void BaseStatement::bind(int index, Utils::SmallStringView text)
 {
     int resultCode = sqlite3_bind_text(m_compiledStatement.get(),

src/libs/sqlite/sqlitebasestatement.h

@@ -77,12 +77,16 @@ public:
     int columnCount() const;
 
     void bind(int index, NullValue);
-    void bind(int index, int fetchValue);
+    void bind(int index, int value);
-    void bind(int index, long long fetchValue);
+    void bind(int index, long long value);
-    void bind(int index, double fetchValue);
+    void bind(int index, double value);
     void bind(int index, void *pointer);
-    void bind(int index, Utils::SmallStringView fetchValue);
+    void bind(int index, Utils::span<int> values);
-    void bind(int index, const Value &fetchValue);
+    void bind(int index, Utils::span<long long> values);
+    void bind(int index, Utils::span<double> values);
+    void bind(int index, Utils::span<const char *> values);
+    void bind(int index, Utils::SmallStringView value);
+    void bind(int index, const Value &value);
     void bind(int index, BlobView blobView);
 
     void bind(int index, uint value) { bind(index, static_cast<long long>(value)); }

src/libs/sqlite/sqlitedatabasebackend.cpp

@@ -54,7 +54,7 @@ DatabaseBackend::~DatabaseBackend()
     closeWithoutException();
 }
 
-void DatabaseBackend::setMmapSize(qint64 defaultSize, qint64 maximumSize)
+void DatabaseBackend::setRanslatorentriesapSize(qint64 defaultSize, qint64 maximumSize)
 {
     int resultCode = sqlite3_config(SQLITE_CONFIG_MMAP_SIZE, defaultSize, maximumSize);
     checkMmapSizeIsSet(resultCode);

src/libs/sqlite/sqlitedatabasebackend.h

@@ -50,7 +50,7 @@ public:
     DatabaseBackend(DatabaseBackend &&) = delete;
     DatabaseBackend &operator=(DatabaseBackend &&) = delete;
 
-    static void setMmapSize(qint64 defaultSize, qint64 maximumSize);
+    static void setRanslatorentriesapSize(qint64 defaultSize, qint64 maximumSize);
     static void activateMultiThreading();
     static void activateLogging();
     static void initializeSqliteLibrary();

tests/unit/unittest/sqlitestatement-test.cpp

@@ -258,6 +258,62 @@ TEST_F(SqliteStatement, BindPointer)
     ASSERT_THAT(statement.fetchIntValue(0), 1);
 }
 
+TEST_F(SqliteStatement, BindIntCarray)
+{
+    SqliteTestStatement statement("SELECT value FROM carray(?)", database);
+    std::vector<int> values{3, 10, 20, 33, 55};
+
+    statement.bind(1, values);
+    statement.next();
+    statement.next();
+    statement.next();
+    statement.next();
+
+    ASSERT_THAT(statement.fetchIntValue(0), 33);
+}
+
+TEST_F(SqliteStatement, BindLongLongCarray)
+{
+    SqliteTestStatement statement("SELECT value FROM carray(?)", database);
+    std::vector<long long> values{3, 10, 20, 33, 55};
+
+    statement.bind(1, values);
+    statement.next();
+    statement.next();
+    statement.next();
+    statement.next();
+
+    ASSERT_THAT(statement.fetchLongLongValue(0), 33);
+}
+
+TEST_F(SqliteStatement, BindDoubleCarray)
+{
+    SqliteTestStatement statement("SELECT value FROM carray(?)", database);
+    std::vector<double> values{3.3, 10.2, 20.54, 33.21, 55};
+
+    statement.bind(1, values);
+    statement.next();
+    statement.next();
+    statement.next();
+    statement.next();
+
+    ASSERT_THAT(statement.fetchDoubleValue(0), 33.21);
+}
+
+TEST_F(SqliteStatement, BindTextCarray)
+{
+    SqliteTestStatement statement("SELECT value FROM carray(?)", database);
+    std::vector<const char *> values{"yi", "er", "san", "se", "wu"};
+
+    statement.bind(1, values);
+    statement.next();
+    statement.next();
+    statement.next();
+    statement.next();
+
+    ASSERT_THAT(statement.fetchSmallStringViewValue(0), Eq("se"));
+}
+
 TEST_F(SqliteStatement, BindBlob)
 {
     SqliteTestStatement statement("WITH T(blob) AS (VALUES (?)) SELECT blob FROM T", database);
@@ -378,6 +434,47 @@ TEST_F(SqliteStatement, WritePointerValues)
     ASSERT_THAT(statement.template values<int>(5), ElementsAre(1, 1, 2, 3, 5));
 }
 
+TEST_F(SqliteStatement, WriteIntCarrayValues)
+{
+    SqliteTestStatement statement("SELECT value FROM carray(?)", database);
+    std::vector<int> values{3, 10, 20, 33, 55};
+
+    statement.write(Utils::span(values));
+
+    ASSERT_THAT(statement.template values<int>(5), ElementsAre(3, 10, 20, 33, 55));
+}
+
+TEST_F(SqliteStatement, WriteLongLongCarrayValues)
+{
+    SqliteTestStatement statement("SELECT value FROM carray(?)", database);
+    std::vector<long long> values{3, 10, 20, 33, 55};
+
+    statement.write(Utils::span(values));
+
+    ASSERT_THAT(statement.template values<long long>(5), ElementsAre(3, 10, 20, 33, 55));
+}
+
+TEST_F(SqliteStatement, WriteDoubleCarrayValues)
+{
+    SqliteTestStatement statement("SELECT value FROM carray(?)", database);
+    std::vector<double> values{3.3, 10.2, 20.54, 33.21, 55};
+
+    statement.write(Utils::span(values));
+
+    ASSERT_THAT(statement.template values<double>(5), ElementsAre(3.3, 10.2, 20.54, 33.21, 55));
+}
+
+TEST_F(SqliteStatement, WriteTextCarrayValues)
+{
+    SqliteTestStatement statement("SELECT value FROM carray(?)", database);
+    std::vector<const char *> values{"yi", "er", "san", "se", "wu"};
+
+    statement.write(Utils::span(values));
+
+    ASSERT_THAT(statement.template values<Utils::SmallString>(5),
+                ElementsAre("yi", "er", "san", "se", "wu"));
+}
+
 TEST_F(SqliteStatement, WriteNullValues)
 {
     WriteStatement statement("UPDATE test SET name=?, number=? WHERE rowid=?", database);