Merge pull request #191 from citusdata/develop_v1x

Merge v1.6.2 changes

.travis.yml

@@ -15,8 +15,9 @@ env:
     - PGVERSION=9.5
     - PGVERSION=9.6
     - PGVERSION=10
+    - PGVERSION=11
 before_install:
-  - git clone -b v0.6.3 --depth 1 https://github.com/citusdata/tools.git
+  - git clone -b v0.7.9 --depth 1 https://github.com/citusdata/tools.git
   - sudo make -C tools install
   - setup_apt
   - nuke_pg

META.json

@@ -2,15 +2,15 @@
    "name": "cstore_fdw",
    "abstract": "Columnar Store for PostgreSQL",
    "description": "PostgreSQL extension which implements a Columnar Store.",
-   "version": "1.6.0",
+   "version": "1.6.2",
    "maintainer": "Murat Tuncer <[email protected]>",
    "license": "apache_2_0",
    "provides": {
       "cstore_fdw": {
          "abstract": "Foreign Data Wrapper for Columnar Store Tables",
          "file": "cstore_fdw--1.6.sql",
          "docfile": "README.md",
-         "version": "1.6.0"
+         "version": "1.6.2"
       }
    },
    "prereqs": {

Makefile

@@ -41,8 +41,8 @@ ifndef MAJORVERSION
     MAJORVERSION := $(basename $(VERSION))
 endif
 
-ifeq (,$(findstring $(MAJORVERSION), 9.3 9.4 9.5 9.6 10))
-    $(error PostgreSQL 9.3 or 9.4 or 9.5 or 9.6 or 10 is required to compile this extension)
+ifeq (,$(findstring $(MAJORVERSION), 9.3 9.4 9.5 9.6 10 11))
+    $(error PostgreSQL 9.3 or 9.4 or 9.5 or 9.6 or 10 or 11 is required to compile this extension)
 endif
 
 cstore.pb-c.c: cstore.proto

README.md

@@ -284,6 +284,8 @@ the installation:
 
 Changeset
 ---------
+### Version 1.6.2
+* (Fix) Add support for PostgreSQL 11
 ### Version 1.6.1
 * (Fix) Fix crash during truncate (Cstore crashing server when enabled, not used)
 * (Fix) No such file or directory warning when attempting to drop database

cstore_fdw.c

@@ -16,6 +16,7 @@
 
 #include "postgres.h"
 #include "cstore_fdw.h"
+#include "cstore_version_compat.h"
 
 #include <sys/stat.h>
 #include <unistd.h>
@@ -57,20 +58,6 @@
 #include "utils/tqual.h"
 
 
-#define PREVIOUS_UTILITY (PreviousProcessUtilityHook != NULL \
-						  ? PreviousProcessUtilityHook : standard_ProcessUtility)
-#if PG_VERSION_NUM >= 100000
-#define CALL_PREVIOUS_UTILITY(parseTree, queryString, context, paramListInfo, \
-							  destReceiver, completionTag) \
-	PREVIOUS_UTILITY(plannedStatement, queryString, context, paramListInfo, \
-					 queryEnvironment, destReceiver, completionTag)
-#else
-#define CALL_PREVIOUS_UTILITY(parseTree, queryString, context, paramListInfo, \
-							  destReceiver, completionTag) \
-	PREVIOUS_UTILITY(parseTree, queryString, context, paramListInfo, destReceiver, \
-					 completionTag)
-#endif
-
 /* local functions forward declarations */
 #if PG_VERSION_NUM >= 100000
 static void CStoreProcessUtility(PlannedStmt *plannedStatement, const char *queryString,
@@ -552,9 +539,7 @@ CopyIntoCStoreTable(const CopyStmt *copyStatement, const char *queryString)
 	 */
 	tupleContext = AllocSetContextCreate(CurrentMemoryContext,
 										 "CStore COPY Row Memory Context",
-										 ALLOCSET_DEFAULT_MINSIZE,
-										 ALLOCSET_DEFAULT_INITSIZE,
-										 ALLOCSET_DEFAULT_MAXSIZE);
+										 ALLOCSET_DEFAULT_SIZES);
 
 	/* init state to read from COPY data source */
 #if (PG_VERSION_NUM >= 100000)
@@ -822,7 +807,7 @@ OpenRelationsForTruncate(List *cstoreTableList)
 											   ACL_TRUNCATE);
 		if (aclresult != ACLCHECK_OK)
 		{
-			aclcheck_error(aclresult, ACL_KIND_CLASS, get_rel_name(relationId));
+			aclcheck_error(aclresult, ACLCHECK_OBJECT_TABLE, get_rel_name(relationId));
 		}
 
 		/* check if this relation is repeated */
@@ -1795,7 +1780,6 @@ ColumnList(RelOptInfo *baserel, Oid foreignTableId)
 	const AttrNumber wholeRow = 0;
 	Relation relation = heap_open(foreignTableId, AccessShareLock);
 	TupleDesc tupleDescriptor = RelationGetDescr(relation);
-	Form_pg_attribute *attributeFormArray = tupleDescriptor->attrs;
 
 	/* first add the columns used in joins and projections */
 	foreach(targetColumnCell, targetColumnList)
@@ -1854,7 +1838,7 @@ ColumnList(RelOptInfo *baserel, Oid foreignTableId)
 			}
 			else if (neededColumn->varattno == wholeRow)
 			{
-				Form_pg_attribute attributeForm = attributeFormArray[columnIndex - 1];
+				Form_pg_attribute attributeForm =  TupleDescAttr(tupleDescriptor, columnIndex - 1);
 				Index tableId = neededColumn->varno;
 
 				column = makeVar(tableId, columnIndex, attributeForm->atttypid,
@@ -1907,8 +1891,8 @@ CStoreBeginForeignScan(ForeignScanState *scanState, int executorFlags)
 	TableReadState *readState = NULL;
 	Oid foreignTableId = InvalidOid;
 	CStoreFdwOptions *cstoreFdwOptions = NULL;
-	TupleTableSlot *tupleSlot = scanState->ss.ss_ScanTupleSlot;
-	TupleDesc tupleDescriptor = tupleSlot->tts_tupleDescriptor;
+	Relation currentRelation = scanState->ss.ss_currentRelation;
+	TupleDesc tupleDescriptor = RelationGetDescr(currentRelation);
 	List *columnList = NIL;
 	ForeignScan *foreignScan = NULL;
 	List *foreignPrivateList = NIL;
@@ -2053,13 +2037,13 @@ CStoreAcquireSampleRows(Relation relation, int logLevel,
 
 	TupleDesc tupleDescriptor = RelationGetDescr(relation);
 	uint32 columnCount = tupleDescriptor->natts;
-	Form_pg_attribute *attributeFormArray = tupleDescriptor->attrs;
+
 
 	/* create list of columns of the relation */
 	uint32 columnIndex = 0;
 	for (columnIndex = 0; columnIndex < columnCount; columnIndex++)
 	{
-		Form_pg_attribute attributeForm = attributeFormArray[columnIndex];
+		Form_pg_attribute attributeForm = TupleDescAttr(tupleDescriptor, columnIndex);
 		const Index tableId = 1;
 
 		if (!attributeForm->attisdropped)
@@ -2078,7 +2062,11 @@ CStoreAcquireSampleRows(Relation relation, int logLevel,
 	/* set up tuple slot */
 	columnValues = palloc0(columnCount * sizeof(Datum));
 	columnNulls = palloc0(columnCount * sizeof(bool));
+#if PG_VERSION_NUM >= 110000
+	scanTupleSlot = MakeTupleTableSlot(NULL);
+#else
 	scanTupleSlot = MakeTupleTableSlot();
+#endif
 	scanTupleSlot->tts_tupleDescriptor = tupleDescriptor;
 	scanTupleSlot->tts_values = columnValues;
 	scanTupleSlot->tts_isnull = columnNulls;
@@ -2095,9 +2083,7 @@ CStoreAcquireSampleRows(Relation relation, int logLevel,
 	 */
 	tupleContext = AllocSetContextCreate(CurrentMemoryContext,
 										 "cstore_fdw temporary context",
-										 ALLOCSET_DEFAULT_MINSIZE,
-										 ALLOCSET_DEFAULT_INITSIZE,
-										 ALLOCSET_DEFAULT_MAXSIZE);
+										 ALLOCSET_DEFAULT_SIZES);
 
 	CStoreBeginForeignScan(scanState, executorFlags);
 

cstore_fdw.h

@@ -312,6 +312,7 @@ extern Datum cstore_ddl_event_end_trigger(PG_FUNCTION_ARGS);
 
 /* Function declarations for utility UDFs */
 extern Datum cstore_table_size(PG_FUNCTION_ARGS);
+extern Datum cstore_clean_table_resources(PG_FUNCTION_ARGS);
 
 /* Function declarations for foreign data wrapper */
 extern Datum cstore_fdw_handler(PG_FUNCTION_ARGS);

cstore_reader.c

@@ -17,6 +17,7 @@
 #include "postgres.h"
 #include "cstore_fdw.h"
 #include "cstore_metadata_serialization.h"
+#include "cstore_version_compat.h"
 
 #include "access/nbtree.h"
 #include "access/skey.h"
@@ -55,7 +56,7 @@ static StripeSkipList * LoadStripeSkipList(FILE *tableFile,
 										   StripeFooter *stripeFooter,
 										   uint32 columnCount,
 										   bool *projectedColumnMask,
-										   Form_pg_attribute *attributeFormArray);
+										   TupleDesc tupleDescriptor);
 static bool * SelectedBlockMask(StripeSkipList *stripeSkipList,
 								List *projectedColumnList, List *whereClauseList);
 static List * BuildRestrictInfoList(List *whereClauseList);
@@ -75,7 +76,6 @@ static void DeserializeDatumArray(StringInfo datumBuffer, bool *existsArray,
 								  int datumTypeLength, char datumTypeAlign,
 								  Datum *datumArray);
 static void DeserializeBlockData(StripeBuffers *stripeBuffers, uint64 blockIndex,
-								 Form_pg_attribute *attributeFormArray,
 								 uint32 rowCount, ColumnBlockData **blockDataArray,
 								 TupleDesc tupleDescriptor);
 static Datum ColumnDefaultValue(TupleConstr *tupleConstraints,
@@ -126,9 +126,7 @@ CStoreBeginRead(const char *filename, TupleDesc tupleDescriptor,
 	 */
 	stripeReadContext = AllocSetContextCreate(CurrentMemoryContext,
 											  "Stripe Read Memory Context",
-											  ALLOCSET_DEFAULT_MINSIZE,
-											  ALLOCSET_DEFAULT_INITSIZE,
-											  ALLOCSET_DEFAULT_MAXSIZE);
+											  ALLOCSET_DEFAULT_SIZES);
 
 	columnCount = tupleDescriptor->natts;
 	projectedColumnMask = ProjectedColumnMask(columnCount, projectedColumnList);
@@ -233,7 +231,6 @@ CStoreReadNextRow(TableReadState *readState, Datum *columnValues, bool *columnNu
 	uint32 blockIndex = 0;
 	uint32 blockRowIndex = 0;
 	TableFooter *tableFooter = readState->tableFooter;
-	Form_pg_attribute *attributeFormArray = readState->tupleDescriptor->attrs;
 	MemoryContext oldContext = NULL;
 
 	/*
@@ -300,7 +297,7 @@ CStoreReadNextRow(TableReadState *readState, Datum *columnValues, bool *columnNu
 
 		oldContext = MemoryContextSwitchTo(readState->stripeReadContext);
 
-		DeserializeBlockData(readState->stripeBuffers, blockIndex, attributeFormArray,
+		DeserializeBlockData(readState->stripeBuffers, blockIndex,
 							 blockRowCount, readState->blockDataArray,
 							 readState->tupleDescriptor);
 
@@ -474,7 +471,6 @@ LoadFilteredStripeBuffers(FILE *tableFile, StripeMetadata *stripeMetadata,
 	ColumnBuffers **columnBuffersArray = NULL;
 	uint64 currentColumnFileOffset = 0;
 	uint32 columnIndex = 0;
-	Form_pg_attribute *attributeFormArray = tupleDescriptor->attrs;
 	uint32 columnCount = tupleDescriptor->natts;
 
 	StripeFooter *stripeFooter = LoadStripeFooter(tableFile, stripeMetadata,
@@ -484,7 +480,7 @@ LoadFilteredStripeBuffers(FILE *tableFile, StripeMetadata *stripeMetadata,
 	StripeSkipList *stripeSkipList = LoadStripeSkipList(tableFile, stripeMetadata,
 														stripeFooter, columnCount,
 														projectedColumnMask,
-														attributeFormArray);
+														tupleDescriptor);
 
 	bool *selectedBlockMask = SelectedBlockMask(stripeSkipList, projectedColumnList,
 												whereClauseList);
@@ -508,7 +504,7 @@ LoadFilteredStripeBuffers(FILE *tableFile, StripeMetadata *stripeMetadata,
 		{
 			ColumnBlockSkipNode *blockSkipNode =
 				selectedBlockSkipList->blockSkipNodeArray[columnIndex];
-			Form_pg_attribute attributeForm = attributeFormArray[columnIndex];
+			Form_pg_attribute attributeForm = TupleDescAttr(tupleDescriptor, columnIndex);
 			uint32 blockCount = selectedBlockSkipList->blockCount;
 
 			ColumnBuffers *columnBuffers = LoadColumnBuffers(tableFile, blockSkipNode,
@@ -649,7 +645,7 @@ static StripeSkipList *
 LoadStripeSkipList(FILE *tableFile, StripeMetadata *stripeMetadata,
 				   StripeFooter *stripeFooter, uint32 columnCount,
 				   bool *projectedColumnMask,
-				   Form_pg_attribute *attributeFormArray)
+				   TupleDesc tupleDescriptor)
 {
 	StripeSkipList *stripeSkipList = NULL;
 	ColumnBlockSkipNode **blockSkipNodeArray = NULL;
@@ -680,7 +676,7 @@ LoadStripeSkipList(FILE *tableFile, StripeMetadata *stripeMetadata,
 		 */
 		if (projectedColumnMask[columnIndex] || firstColumn)
 		{
-			Form_pg_attribute attributeForm = attributeFormArray[columnIndex];
+			Form_pg_attribute attributeForm = TupleDescAttr(tupleDescriptor, columnIndex);
 
 			StringInfo columnSkipListBuffer =
 				ReadFromFile(tableFile, currentColumnSkipListFileOffset,
@@ -1174,14 +1170,14 @@ DeserializeDatumArray(StringInfo datumBuffer, bool *existsArray, uint32 datumCou
  */
 static void
 DeserializeBlockData(StripeBuffers *stripeBuffers, uint64 blockIndex,
-					 Form_pg_attribute *attributeFormArray, uint32 rowCount,
+					 uint32 rowCount,
 					 ColumnBlockData **blockDataArray, TupleDesc tupleDescriptor)
 {
 	int columnIndex = 0;
 	for (columnIndex = 0; columnIndex < stripeBuffers->columnCount; columnIndex++)
 	{
 		ColumnBlockData *blockData = blockDataArray[columnIndex];
-		Form_pg_attribute attributeForm = attributeFormArray[columnIndex];
+		Form_pg_attribute attributeForm = TupleDescAttr(tupleDescriptor, columnIndex);
 		ColumnBuffers *columnBuffers = stripeBuffers->columnBuffersArray[columnIndex];
 		bool columnAdded = false;
 

cstore_version_compat.h

@@ -0,0 +1,50 @@
+/*-------------------------------------------------------------------------
+ *
+ * cstore_version_compat.h
+ *
+ * 	Compatibility macros for writing code agnostic to PostgreSQL versions
+ *
+ * Copyright (c) 2018, Citus Data, Inc.
+ *
+ * $Id$
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef CSTORE_COMPAT_H
+#define CSTORE_COMPAT_H
+
+#if PG_VERSION_NUM < 100000
+
+/* Accessor for the i'th attribute of tupdesc. */
+#define TupleDescAttr(tupdesc, i) ((tupdesc)->attrs[(i)])
+
+#endif
+
+#if PG_VERSION_NUM < 110000
+#define ALLOCSET_DEFAULT_SIZES ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE
+#define ACLCHECK_OBJECT_TABLE ACL_KIND_CLASS
+#else
+#define ACLCHECK_OBJECT_TABLE OBJECT_TABLE
+
+#define ExplainPropertyLong(qlabel, value, es) \
+	ExplainPropertyInteger(qlabel, NULL, value, es)
+#endif
+
+#define PREVIOUS_UTILITY (PreviousProcessUtilityHook != NULL \
+						  ? PreviousProcessUtilityHook : standard_ProcessUtility)
+#if PG_VERSION_NUM >= 100000
+#define CALL_PREVIOUS_UTILITY(parseTree, queryString, context, paramListInfo, \
+							  destReceiver, completionTag) \
+	PREVIOUS_UTILITY(plannedStatement, queryString, context, paramListInfo, \
+					 queryEnvironment, destReceiver, completionTag)
+#else
+#define CALL_PREVIOUS_UTILITY(parseTree, queryString, context, paramListInfo, \
+							  destReceiver, completionTag) \
+	PREVIOUS_UTILITY(parseTree, queryString, context, paramListInfo, destReceiver, \
+					 completionTag)
+#endif
+
+
+
+#endif /* CSTORE_COMPAT_H */

cstore_writer.c

@@ -17,6 +17,7 @@
 #include "postgres.h"
 #include "cstore_fdw.h"
 #include "cstore_metadata_serialization.h"
+#include "cstore_version_compat.h"
 
 #include <sys/stat.h>
 #include "access/nbtree.h"
@@ -149,7 +150,7 @@ CStoreBeginWrite(const char *filename, CompressionType compressionType,
 	for (columnIndex = 0; columnIndex < columnCount; columnIndex++)
 	{
 		FmgrInfo *comparisonFunction = NULL;
-		FormData_pg_attribute *attributeForm = tupleDescriptor->attrs[columnIndex];
+		FormData_pg_attribute *attributeForm = TupleDescAttr(tupleDescriptor, columnIndex);
 
 		if (!attributeForm->attisdropped)
 		{
@@ -168,9 +169,7 @@ CStoreBeginWrite(const char *filename, CompressionType compressionType,
 	 */
 	stripeWriteContext = AllocSetContextCreate(CurrentMemoryContext,
 											   "Stripe Write Memory Context",
-											   ALLOCSET_DEFAULT_MINSIZE,
-											   ALLOCSET_DEFAULT_INITSIZE,
-											   ALLOCSET_DEFAULT_MAXSIZE);
+											   ALLOCSET_DEFAULT_SIZES);
 
 	columnMaskArray = palloc(columnCount * sizeof(bool));
 	memset(columnMaskArray, true, columnCount);
@@ -258,7 +257,7 @@ CStoreWriteRow(TableWriteState *writeState, Datum *columnValues, bool *columnNul
 			FmgrInfo *comparisonFunction =
 				writeState->comparisonFunctionArray[columnIndex];
 			Form_pg_attribute attributeForm =
-				writeState->tupleDescriptor->attrs[columnIndex];
+				TupleDescAttr(writeState->tupleDescriptor, columnIndex);
 			bool columnTypeByValue = attributeForm->attbyval;
 			int columnTypeLength = attributeForm->attlen;
 			Oid columnCollation = attributeForm->attcollation;
@@ -640,7 +639,7 @@ CreateSkipListBufferArray(StripeSkipList *stripeSkipList, TupleDesc tupleDescrip
 		StringInfo skipListBuffer = NULL;
 		ColumnBlockSkipNode *blockSkipNodeArray =
 			stripeSkipList->blockSkipNodeArray[columnIndex];
-		Form_pg_attribute attributeForm = tupleDescriptor->attrs[columnIndex];
+		Form_pg_attribute attributeForm = TupleDescAttr(tupleDescriptor, columnIndex);
 
 		skipListBuffer = SerializeColumnSkipList(blockSkipNodeArray,
 												 stripeSkipList->blockCount,