diff --git a/src/Runtime/XSharp.SQLRdd/Classes/Command.prg b/src/Runtime/XSharp.SQLRdd/Classes/Command.prg
index 0fa68e1e5b..712e83ab44 100644
--- a/src/Runtime/XSharp.SQLRdd/Classes/Command.prg
+++ b/src/Runtime/XSharp.SQLRdd/Classes/Command.prg
@@ -150,6 +150,58 @@ class SqlDbCommand inherit SqlDbHandleObject implements IDisposable
end try
end method
+ ///
+ /// Execute the command and return the rows in a copy of oTemplate's structure.
+ ///
+ /// The name of the DataTable (when the template cannot be used)
+ /// An empty table with the columns the command returns, or NULL
+ ///
+ /// Building the structure from the reader's schema (DataTable.Load()) is the expensive part of
+ /// a query that returns one or a few rows of a wide table: 1.5 ms against 0.5 ms for 178
+ /// columns. The RDD runs every query of a work area with the same column list, so it passes
+ /// the structure of its buffer. When the reader's columns do not match the template, the
+ /// rows are loaded as by GetDataTable(cTable).
+ ///
+ method GetDataTable(cTable as string, oTemplate as DataTable) as DataTable
+ if oTemplate == null
+ return SELF:GetDataTable(cTable)
+ endif
+ try
+ using var reader := SELF:ExecuteReader(cTable)
+ if reader == null
+ return null
+ endif
+ var lMatch := reader:FieldCount == oTemplate:Columns:Count
+ for var i := 0 to reader:FieldCount-1
+ if !lMatch
+ exit
+ endif
+ lMatch := String.Equals(reader:GetName(i), oTemplate:Columns[i]:ColumnName, StringComparison.OrdinalIgnoreCase)
+ next
+ local oDataTable as DataTable
+ if lMatch
+ oDataTable := oTemplate:Clone()
+ var aValues := object[]{reader:FieldCount}
+ oDataTable:BeginLoadData()
+ do while reader:Read()
+ reader:GetValues(aValues)
+ oDataTable:LoadDataRow(aValues, true)
+ enddo
+ oDataTable:EndLoadData()
+ else
+ oDataTable := DataTable{iif(String.IsNullOrEmpty(cTable), SELF:Name, cTable)}
+ var oDataSet := DataSet{}
+ oDataSet:EnforceConstraints := false
+ oDataSet:Tables:Add(oDataTable)
+ oDataTable:Load(reader)
+ endif
+ return oDataTable
+ catch e as Exception
+ Connection:LastException := e
+ return null
+ end try
+ end method
+
///
/// Execute a scalar command
///
diff --git a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg
index 7257f8d665..61ef6ab3c5 100644
--- a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg
+++ b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg
@@ -616,6 +616,11 @@ partial class SQLRDD inherit Workarea
if Connection?:Provider is null .or. !self:Connection:IsOpen .or. _oTd == null
return recNos:ToArray()
endif
+ if !self:Connection:UseLocks
+ // No lock table entries are ever written without UseLocks. Code that asks for the
+ // lock list after every movement (RLockList/IsLocked()) paid a query each time.
+ return recNos:ToArray()
+ endif
try
var sb := StringBuilder{}
sb:AppendLine("select recno from " + SqlDbConnection.LockTableName)
@@ -776,7 +781,7 @@ partial class SQLRDD inherit Workarea
IF nRec == 0
// DBF semantics: GoTo(0) positions on the phantom record (EOF) without any I/O.
// Browsers do this for every empty row they paint, so it must not cost a query.
- SELF:_GoToPhantom()
+ SELF:_GoToPhantom(TRUE)
RETURN TRUE
ENDIF
SELF:_eofPending := FALSE
@@ -804,7 +809,27 @@ partial class SQLRDD inherit Workarea
next
endif
- var found := SELF:_GotoRecord(nRec)
+ // Code that saves the record number, seeks elsewhere and goes back (or follows a
+ // relation) mostly just reads the record's fields afterwards. Its position in the order
+ // is only needed when something moves from it (Skip(), OrderKeyNo ...), see
+ // _EnsurePositioned(): load just the record by its recno. Looking the position up right
+ // away cost a ROW_NUMBER() over the order plus a whole page around the record.
+ // When the previous GoTo() already loaded just one record this way, position fully: a
+ // browse that calls GoTo() for every row it paints then gets a page with the neighbouring
+ // rows instead of one query per row. (A single record left by an order change does not
+ // count - SetOrder() followed by GoTo(saved record) is the usual restore pattern.)
+ local found as logic
+ if SELF:_tableMode == TableMode.Table .and. SELF:_recnoColumNo >= 0 .and. SELF:DataTable != null ;
+ .and. !(SELF:_lazyGoto .and. !SELF:_bufferContiguous .and. SELF:RowCount <= 1)
+ found := SELF:_LoadSingleRecord(nRec)
+ if found
+ SELF:_lazyGoto := TRUE
+ else
+ found := SELF:_GotoRecord(nRec)
+ endif
+ else
+ found := SELF:_GotoRecord(nRec)
+ endif
SELF:_CheckEofBof()
RETURN found
end method
diff --git a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Orders.prg b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Orders.prg
index 86331e7d89..61f80f81d8 100644
--- a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Orders.prg
+++ b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Orders.prg
@@ -195,6 +195,13 @@ partial class SQLRDD
// index/order" pattern used throughout the app, and it must keep flushing even when
// the order doesn't actually change.
SELF:GoCold()
+ // The record we stand on (just written by GoCold()) is kept across the order change
+ // instead of being read again, and LastRec() does not change with the order.
+ local aCurrentRow := null as object[]
+ if !lWasEof .and. SELF:_hasData .and. SELF:DataTable != null .and. SELF:RowNumber >= 1 .and. SELF:RowNumber <= SELF:RowCount
+ aCurrentRow := SELF:CurrentRow:ItemArray
+ endif
+ var lTableCountValid := SELF:_tableReccountValid
if (orderInfo:Order is long var nOrder0 .and. nOrder0 == 0) .or. ;
(orderInfo:Order is string var cOrder0 .and. String.IsNullOrEmpty(cOrder0))
// Order 0 (or an empty order name) means: switch back to the natural/physical
@@ -211,8 +218,9 @@ partial class SQLRDD
self:_CloseCursor()
endif
SELF:CurrentOrder := null
+ self:_tableReccountValid := lTableCountValid
if lReset
- self:_RepositionAfterOrderChange(currentRecord, lWasEof)
+ self:_RepositionAfterOrderChange(currentRecord, lWasEof, aCurrentRow)
else
self:GoTo(currentRecord)
endif
@@ -224,6 +232,7 @@ partial class SQLRDD
self:_CloseCursor()
endif
SELF:CurrentOrder := newOrder
+ self:_tableReccountValid := lTableCountValid
result := CurrentOrder != null
IF result
SELF:CurrentOrder:ClearCache()
@@ -232,7 +241,7 @@ partial class SQLRDD
// loaded row. _CloseCursor() above cleared it, so without this there would
// be nothing to evaluate.
if lReset
- self:_RepositionAfterOrderChange(currentRecord, lWasEof)
+ self:_RepositionAfterOrderChange(currentRecord, lWasEof, aCurrentRow)
else
self:GoTo(currentRecord)
endif
@@ -569,7 +578,12 @@ partial class SQLRDD
// null instead of an empty table - treat that the same as "no rows found" instead of
// crashing on DataTable:Rows below, same fix as GoTo()/_ClearTable() already got.
IF (SELF:DataTable == null .or. SELF:DataTable:Rows:Count = 0) .and. !seekInfo.SoftSeek
- SELF:GoTo(0)
+ // BOF is set below, so no need to count the order for it (GoTo(0) would)
+ IF SELF:DataTable == null
+ SELF:GoTo(0)
+ ELSE
+ SELF:_GoToPhantom(FALSE)
+ ENDIF
SELF:_Found := false
SELF:_SetEOF(true)
SELF:_SetBOF(false)
diff --git a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg
index 61d6302fd1..336b6fabc5 100644
--- a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg
+++ b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg
@@ -46,12 +46,18 @@ partial class SQLRDD
/// the buffer edges then has to re-locate the current record first.
///
private _bufferContiguous as logic
+ /// Empty table with the buffer's columns, to load further query results into.
+ private _tableTemplate as DataTable
///
/// Rows to fetch with the next forward page (0 = PageSize): doubles with every full page read
/// on in a row (up to BufferSize pages). After a seek it starts small, because code that
/// seeks and then skips while the key matches mostly reads only a few records.
///
private _readAheadRows := 0 as long
+ /// Rows to fetch with the next backward page (0 = PageSize), grows like _readAheadRows.
+ private _readBackRows := 0 as long
+ /// TRUE when the single buffered record was loaded by GoTo() without its position.
+ private _lazyGoto as logic
///
/// TRUE after GoTo(0) positioned on the phantom record while the buffer does not end at the
/// last record of the order: a backward Skip() then has to start from GoBottom().
@@ -132,6 +138,9 @@ partial class SQLRDD
end property
/// Maximum number of rows kept in the local buffer.
private property _MaxBufferRows as long => _oTd:PageSize * Math.Max(_oTd:BufferSize, 1)
+ /// Rows for a first read where only a few records are expected to be needed (after a seek,
+ /// GoBottom(), looking up the position of a record). Reading on makes the pages grow again.
+ private property _SmallPageRows as long => Math.Max(1, _oTd:PageSize / 10)
internal property DataTable as DataTable
get
return _table
@@ -177,6 +186,11 @@ partial class SQLRDD
endif
dbColumn:Flags := DBFFieldFlags.None
next
+ if SELF:_tableTemplate == null
+ // the structure for all further queries of this work area (same column list),
+ // see SqlDbCommand.GetDataTable(cTable, oTemplate)
+ SELF:_tableTemplate := _table:Clone()
+ endif
SELF:RowNumber := 1
SELF:_CheckEofBof()
end set
@@ -613,9 +627,9 @@ partial class SQLRDD
/// See SqlDbTableCommandBuilder.BuildLastPageStatement for why this exists.
private method _FetchLastPage(nTotalRows as long) as logic
try
- SELF:_command:CommandText := _builder:BuildLastPageStatement()
+ SELF:_command:CommandText := _builder:BuildLastPageStatement(SELF:_SmallPageRows)
SELF:_command:ClearParameters()
- var newTable := SELF:_command:GetDataTable(SELF:Alias)
+ var newTable := SELF:_command:GetDataTable(SELF:Alias, SELF:_tableTemplate)
if newTable == null
return false
endif
@@ -632,6 +646,8 @@ partial class SQLRDD
SELF:_anchorWhere := ""
SELF:_bufferContiguous := true
SELF:_bufferStart := Math.Max(0, nTotalRows - SELF:RowCount)
+ // reading backwards from the bottom starts small as well
+ SELF:_readBackRows := SELF:_SmallPageRows
// This IS the last page by definition - without this, a Skip(1) right after
// GoBottom() (e.g. GoTo(0)'s GoBottom()+Skip(1)) would not know it's already at
// the end, and would fall through to fetching "the next page" via the normal
@@ -656,11 +672,11 @@ partial class SQLRDD
return self:_OpenTable("", "")
end method
- private method _ReadTable(sWhereClause as string, nOffset as long, nCount as long) as DataTable
+ private method _ReadTable(sWhereClause as string, nOffset as long, nCount as long, lReverse := false as logic) as DataTable
try
- _command:CommandText := self:_BuildSqlStatement(sWhereClause, nOffset, nCount)
+ _command:CommandText := self:_BuildSqlStatement(sWhereClause, nOffset, nCount, lReverse)
_command:ClearParameters()
- var oDataTable := _command:GetDataTable(self:Alias)
+ var oDataTable := _command:GetDataTable(self:Alias, SELF:_tableTemplate)
return oDataTable
catch as Exception
return null
@@ -679,7 +695,8 @@ partial class SQLRDD
SELF:_anchorWhere := sAnchorWhere
SELF:_bufferContiguous := true
// a hard seek (sLoadWhere narrower than the anchor) mostly continues with a few records
- SELF:_readAheadRows := iif(sLoadWhere != sAnchorWhere .and. !String.IsNullOrEmpty(sAnchorWhere), Math.Max(1, _oTd:PageSize / 10), 0)
+ SELF:_readAheadRows := iif(sLoadWhere != sAnchorWhere .and. !String.IsNullOrEmpty(sAnchorWhere), SELF:_SmallPageRows, 0)
+ SELF:_readBackRows := 0
// A fresh open/reposition (e.g. from Seek()) must not inherit a stale "no more
// rows" flag left over from whatever this buffer was doing before - otherwise
// Skip() refuses to fetch the next page and reports EOF even though the new
@@ -710,10 +727,10 @@ partial class SQLRDD
end method
- private method _BuildSqlStatement(sWhereClause as string, nOffset as long, nCount as long) as string
+ private method _BuildSqlStatement(sWhereClause as string, nOffset as long, nCount as long, lReverse := false as logic) as string
local query as string
if self:_tableMode == TableMode.Table
- query := _builder:BuildSqlStatement(sWhereClause, (int) nOffset, (int) nCount)
+ query := _builder:BuildSqlStatement(sWhereClause, (int) nOffset, (int) nCount, lReverse)
else
query := self:_oTd:SelectStatement
endif
@@ -779,13 +796,17 @@ partial class SQLRDD
end method
/// Position on the phantom record (EOF) without reading anything.
- private method _GoToPhantom() as void
+ /// FALSE when the caller sets BOF itself, which saves counting the
+ /// order for an empty buffer (e.g. a seek that found nothing).
+ private method _GoToPhantom(lSetBof as logic) as void
SELF:_Found := FALSE
SELF:_outsideOrder := FALSE
SELF:RowNumber := SELF:RowCount + 1
SELF:_SetEOF(TRUE)
// BOF too only for an empty table (an empty buffer alone does not mean that)
- SELF:_SetBOF(SELF:RowCount == 0 .and. SELF:_OrderRowCount() == 0)
+ if lSetBof
+ SELF:_SetBOF(SELF:RowCount == 0 .and. SELF:_OrderRowCount() == 0)
+ endif
// When the buffer already ends at the last record, a backward Skip() from the phantom
// row naturally lands on it; otherwise remember to go to the bottom first.
SELF:_eofPending := !(SELF:_hasEOF .and. SELF:_bufferContiguous)
@@ -851,22 +872,36 @@ partial class SQLRDD
/// Condition for the rows that follow the last buffered row in the current order (keyset
/// paging), or NULL when that cannot be expressed and the caller must page with OFFSET.
///
+ private method _KeysetAfterLastRow() as string
+ if SELF:RowCount == 0 .or. !SELF:_bufferContiguous
+ return null
+ endif
+ return SELF:_KeysetCondition(SELF:DataTable:Rows[SELF:RowCount-1], FALSE, FALSE)
+ end method
+
+ ///
+ /// Condition for the rows that come after oRow (lBefore FALSE) or before it (lBefore TRUE) in
+ /// the current order, including oRow itself when lInclusive is TRUE. NULL when that cannot be
+ /// expressed and the caller must use row offsets instead.
+ ///
///
/// The order is the ORDER BY list of the current order plus the recno column, exactly as
/// SqlDbTableCommandBuilder.BuildSqlStatement() sorts. Every item is compared with its value
- /// for the last row, read by a subquery on that row's recno, so the comparison uses the same
- /// expressions and collation as the ORDER BY (also for keys with functions):
+ /// for oRow (see _KeysetLiteral(), otherwise read by a subquery on the row's recno, so the
+ /// comparison uses the same expressions and collation as the ORDER BY, also for keys with
+ /// functions; lLiteralOnly returns NULL instead, see _KeysetLiteral() for the cost). For the
+ /// rows after oRow:
/// k1 >= v1 AND ( k1 > v1 OR (k1 = v1 AND k2 > v2) OR ... OR (k1 = v1 AND ... AND recno > n) )
- /// The leading bound lets the server seek in an index on k1. A descending item compares with
- /// "<" and also accepts NULL, which sorts after all values in descending order. When the last
- /// row has NULL in one of the key columns its position cannot be expressed this way.
+ /// The leading bound lets the server seek in an index on k1. Where the order goes towards
+ /// lower values (a descending item after oRow, an ascending one before it) the comparison is
+ /// "<" and also accepts NULL, which sorts below all values. When oRow has NULL in one of the
+ /// key columns its position cannot be expressed this way.
///
- private method _KeysetAfterLastRow() as string
- if !SELF:_oTd:HasRecnoColumn .or. SELF:RowCount == 0 .or. !SELF:_bufferContiguous .or. SELF:_recnoColumNo < 0
+ private method _KeysetCondition(oRow as DataRow, lBefore as logic, lInclusive as logic, lLiteralOnly := false as logic) as string
+ if !SELF:_oTd:HasRecnoColumn .or. SELF:_recnoColumNo < 0
return null
endif
- var lastRow := SELF:DataTable:Rows[SELF:RowCount-1]
- var oRecno := lastRow[(int) SELF:_recnoColumNo]
+ var oRecno := oRow[(int) SELF:_recnoColumNo]
if oRecno == null .or. oRecno == DBNull.Value
return null
endif
@@ -879,7 +914,7 @@ partial class SQLRDD
if oOrder != null
foreach var cCol in oOrder:ColumnList
var nCol := SELF:DataTable:Columns:IndexOf(cCol:Trim({'[',']','"','`'}))
- if nCol < 0 .or. lastRow[nCol] == DBNull.Value
+ if nCol < 0 .or. oRow[nCol] == DBNull.Value
return null
endif
next
@@ -897,18 +932,26 @@ partial class SQLRDD
if cExpr == cRecno
aValue:Add(nRecno:ToString())
else
- aValue:Add("(SELECT " + cExpr + " FROM " + cTable + " WHERE " + cRecno + " = " + nRecno:ToString() + ")")
+ var cLiteral := SELF:_KeysetLiteral(oRow, cExpr)
+ if cLiteral == null
+ if lLiteralOnly
+ return null
+ endif
+ cLiteral := "(SELECT " + cExpr + " FROM " + cTable + " WHERE " + cRecno + " = " + nRecno:ToString() + ")"
+ endif
+ aValue:Add(cLiteral)
endif
next
+ var nLast := aExpr:Count - 1
var sb := StringBuilder{}
sb:Append("(")
if aExpr:Count > 1
- sb:Append(SELF:_KeysetCompare(aExpr[0], aDesc[0], aValue[0], TRUE))
+ sb:Append(SELF:_KeysetCompare(aExpr[0], aDesc[0] != lBefore, aValue[0], TRUE))
sb:Append(SqlDbProvider.AndClause)
endif
sb:Append("(")
- for var i := 0 upto aExpr:Count-1
+ for var i := 0 upto nLast
if i > 0
sb:Append(SqlDbProvider.OrClause)
endif
@@ -916,16 +959,53 @@ partial class SQLRDD
for var j := 0 upto i-1
sb:Append(aExpr[j] + " = " + aValue[j] + SqlDbProvider.AndClause)
next
- sb:Append(SELF:_KeysetCompare(aExpr[i], aDesc[i], aValue[i], FALSE))
+ sb:Append(SELF:_KeysetCompare(aExpr[i], aDesc[i] != lBefore, aValue[i], lInclusive .and. i == nLast))
sb:Append(")")
next
sb:Append("))")
return sb:ToString()
end method
- private method _KeysetCompare(cExpr as string, lDesc as logic, cValue as string, lInclusive as logic) as string
- var cComp := iif(lDesc, " <", " >") + iif(lInclusive, "= ", " ")
- if lDesc
+ ///
+ /// The value of the order item cExpr in oRow as a SQL literal, or NULL when the subquery must
+ /// be used: cExpr is not a plain column, or the value cannot be written as a literal that
+ /// compares exactly like the stored value.
+ ///
+ ///
+ /// Every subquery has to be optimized by the server: with one per item, compiling the keyset
+ /// condition of a two column key took 30-45 ms, executing it 3 ms. Since the text changes with
+ /// every page the plan is never reused. The buffer holds the values as read from the server, so
+ /// a literal is exact for strings (ASCII only, so no code page conversion can change them) and
+ /// integers/decimals.
+ ///
+ private method _KeysetLiteral(oRow as DataRow, cExpr as string) as string
+ var nCol := SELF:DataTable:Columns:IndexOf(cExpr:Trim({'[',']','"','`'}))
+ if nCol < 0
+ return null
+ endif
+ switch oRow[nCol]
+ case cValue as string
+ foreach var c in cValue
+ if c > (char) 127
+ return null
+ endif
+ next
+ return Functions.XsValueToSqlValue(cValue)
+ case nValue as long
+ return nValue:ToString(System.Globalization.CultureInfo.InvariantCulture)
+ case nValue as int64
+ return nValue:ToString(System.Globalization.CultureInfo.InvariantCulture)
+ case nValue as short
+ return nValue:ToString(System.Globalization.CultureInfo.InvariantCulture)
+ case nValue as decimal
+ return nValue:ToString(System.Globalization.CultureInfo.InvariantCulture)
+ end switch
+ return null
+ end method
+
+ private method _KeysetCompare(cExpr as string, lLower as logic, cValue as string, lInclusive as logic) as string
+ var cComp := iif(lLower, " <", " >") + iif(lInclusive, "= ", " ")
+ if lLower
return "(" + cExpr + cComp + cValue + SqlDbProvider.OrClause + cExpr + " IS NULL)"
endif
return cExpr + cComp + cValue
@@ -959,6 +1039,9 @@ partial class SQLRDD
nCount := iif(nWanted <= nCount, nWanted, Math.Max(nCount, Math.Min(nWanted, nMax)))
newTable := SELF:_ReadTable(cWhere, 0, nCount)
else
+ if SELF:_readAheadRows > 0 .and. SELF:_readAheadRows < nCount
+ nCount := SELF:_readAheadRows
+ endif
newTable := SELF:_ReadTable(SELF:_anchorWhere, nOffset, nCount)
endif
if newTable == null
@@ -1000,12 +1083,17 @@ partial class SQLRDD
/// buffer, evicting rows from the end when the buffer would grow beyond _MaxBufferRows.
/// RowNumber keeps pointing at the same row. Returns the number of rows fetched.
///
- private method _FetchBackward() as long
+ /// Rows that must at least be read (when there are that many).
+ private method _FetchBackward(nNeeded := 0 as long) as long
SELF:_readAheadRows := 0
if self:_tableMode != TableMode.Table .or. SELF:_bufferStart <= 0
return 0
endif
- var nStart := Math.Max(0, SELF:_bufferStart - _oTd:PageSize)
+ var nKeyset := SELF:_FetchBackwardKeyset(nNeeded)
+ if nKeyset >= 0
+ return nKeyset
+ endif
+ var nStart := Math.Max(0, SELF:_bufferStart - Math.Max(_oTd:PageSize, nNeeded))
var newTable := SELF:_ReadTable(SELF:_anchorWhere, nStart, SELF:_bufferStart - nStart)
if newTable == null
return 0
@@ -1020,6 +1108,63 @@ partial class SQLRDD
next
SELF:_bufferStart := nStart
SELF:RowNumber += nRows
+ SELF:_TrimBufferEnd()
+ return nRows
+
+ ///
+ /// Like _FetchBackward(), but finds the rows before the buffer by the key of its first row
+ /// (sorted in reverse) instead of by their offset, which makes the server read through all
+ /// rows before them. Works on a seek-anchored buffer too, which is then anchored at its new
+ /// first row. Returns -1 when the key cannot be expressed (see _KeysetCondition()).
+ ///
+ private method _FetchBackwardKeyset(nNeeded as long) as long
+ if self:_tableMode != TableMode.Table .or. SELF:RowCount == 0 .or. !SELF:_bufferContiguous
+ return -1
+ endif
+ var cBefore := SELF:_KeysetCondition(SELF:DataTable:Rows[0], TRUE, FALSE, TRUE)
+ if cBefore == null
+ return -1
+ endif
+ local nCount := iif(SELF:_readBackRows > 0, SELF:_readBackRows, _oTd:PageSize) as long
+ nCount := Math.Max(nCount, nNeeded)
+ var newTable := SELF:_ReadTable(cBefore, 0, nCount, TRUE)
+ if newTable == null
+ return -1
+ endif
+ var nRows := newTable:Rows:Count
+ local cAnchor := "" as string
+ if nRows == nCount .and. !String.IsNullOrEmpty(SELF:_anchorWhere)
+ // The rows before the anchor are not part of the anchored result set.
+ cAnchor := SELF:_KeysetCondition(newTable:Rows[nRows-1], FALSE, TRUE, TRUE)
+ if cAnchor == null
+ return -1
+ endif
+ endif
+ // The rows come nearest first.
+ for var nRow := 0 upto nRows-1
+ var newRow := SELF:DataTable:NewRow()
+ newRow:ItemArray := newTable:Rows[nRow]:ItemArray
+ SELF:DataTable:Rows:InsertAt(newRow, 0)
+ newRow:AcceptChanges()
+ next
+ SELF:RowNumber += nRows
+ SELF:_readAheadRows := 0
+ SELF:_readBackRows := iif(nRows < nCount, 0, Math.Min(nCount * 2, SELF:_MaxBufferRows))
+ if nRows < nCount
+ // the buffer now starts at the first row of the order
+ SELF:_anchorWhere := ""
+ SELF:_bufferStart := 0
+ elseif !String.IsNullOrEmpty(SELF:_anchorWhere)
+ SELF:_anchorWhere := cAnchor
+ SELF:_bufferStart := 0
+ else
+ SELF:_bufferStart := Math.Max(0, SELF:_bufferStart - nRows)
+ endif
+ SELF:_TrimBufferEnd()
+ return nRows
+
+ /// Evict rows from the end when the buffer grew beyond _MaxBufferRows.
+ private method _TrimBufferEnd() as void
var nRemove := SELF:RowCount - SELF:_MaxBufferRows
if nRemove > 0
for var i := 1 upto nRemove
@@ -1027,13 +1172,15 @@ partial class SQLRDD
next
SELF:_hasEOF := false
endif
- return nRows
+ return
///
/// Replace the buffer with up to one page of the anchored result set starting at nOffset.
///
- private method _LoadBufferAt(nOffset as long) as long
- SELF:_readAheadRows := 0
+ /// Rows to read, 0 = PageSize.
+ private method _LoadBufferAt(nOffset as long, nRows := 0 as long) as long
+ SELF:_readAheadRows := nRows
+ SELF:_readBackRows := nRows
SELF:_ClearTable()
SELF:_bufferStart := Math.Max(0, nOffset)
SELF:_bufferContiguous := true
@@ -1083,6 +1230,10 @@ partial class SQLRDD
nNewRow := 1
exit
endif
+ // a read-ahead that started small must still reach the target in one page
+ if SELF:_readAheadRows > 0 .and. SELF:_readAheadRows < nNewRow - SELF:RowCount
+ SELF:_readAheadRows := nNewRow - SELF:RowCount
+ endif
if SELF:_FetchForward() == 0
SELF:_hasEOF := true
endif
@@ -1095,6 +1246,18 @@ partial class SQLRDD
SELF:_SetBOF(TRUE)
return true
endif
+ if !String.IsNullOrEmpty(SELF:_anchorWhere) .and. 1 - nNewRow <= _oTd:PageSize
+ // Read the rows before an anchored buffer by key, without looking up its position.
+ var nFetched := SELF:_FetchBackwardKeyset(1 - nNewRow)
+ if nFetched == 0
+ SELF:RowNumber := 1
+ SELF:_SetBOF(TRUE)
+ return true
+ elseif nFetched > 0
+ nNewRow += nFetched
+ loop
+ endif
+ endif
if !String.IsNullOrEmpty(SELF:_anchorWhere)
// Seek-anchored buffer: the rows before the anchor are not part of the anchored
// result set, so switch to absolute positions first.
@@ -1128,12 +1291,13 @@ partial class SQLRDD
nNewRow := 1
exit
endif
- if SELF:_FetchBackward() == 0
+ var nRows := SELF:_FetchBackward(1 - nNewRow)
+ if nRows == 0
SELF:RowNumber := 1
SELF:_SetBOF(TRUE)
return true
endif
- nNewRow := nTarget - SELF:_bufferStart + 1
+ nNewRow += nRows
enddo
SELF:RowNumber := nNewRow
SELF:_SetEOF(FALSE)
@@ -1205,6 +1369,13 @@ partial class SQLRDD
return true
PRIVATE METHOD _GotoRecord(nRec as DWORD) AS LOGIC
+ if SELF:_tableMode == TableMode.Table .and. ;
+ !(SELF:_bufferContiguous .and. String.IsNullOrEmpty(SELF:_anchorWhere) .and. SELF:RowCount > 0) ;
+ .and. SELF:_LoadWindowAt(nRec, -1)
+ // Nothing that could be extended: the buffer now starts at the record, anchored at its
+ // key, and its position in the order is only looked up when needed.
+ RETURN TRUE
+ endif
// GoTo() only calls _GotoRecord() after it already established that nRec is NOT in
// the currently loaded buffer, so its position in the order must be looked up.
var nOffset := SELF:_GetAbsoluteOffset(nRec)
@@ -1227,16 +1398,20 @@ partial class SQLRDD
if SELF:_bufferContiguous .and. String.IsNullOrEmpty(SELF:_anchorWhere) .and. SELF:RowCount > 0
var nBufferEnd := SELF:_bufferStart + SELF:RowCount
if nOffset >= nBufferEnd .and. nOffset < nBufferEnd + _oTd:PageSize .and. !SELF:_hasEOF
+ // the page must reach the record, a smaller read-ahead would stop short of it
+ if SELF:_readAheadRows > 0 .and. SELF:_readAheadRows <= nOffset - nBufferEnd
+ SELF:_readAheadRows := 0
+ endif
lLoaded := SELF:_FetchForward() > 0
elseif nOffset < SELF:_bufferStart .and. nOffset >= SELF:_bufferStart - _oTd:PageSize
- lLoaded := SELF:_FetchBackward() > 0
+ lLoaded := SELF:_FetchBackward(SELF:_bufferStart - nOffset) > 0
endif
endif
- if !lLoaded
- // Center the record in the new page, so that its neighbours on either side
- // are buffered as well.
+ if !lLoaded .and. !SELF:_LoadWindowAt(nRec, nOffset)
+ // Center the record in a small page, so that its neighbours on either side are
+ // buffered as well.
SELF:_anchorWhere := ""
- SELF:_LoadBufferAt(nOffset - _oTd:PageSize / 2)
+ SELF:_LoadBufferAt(nOffset - SELF:_SmallPageRows / 2, SELF:_SmallPageRows)
endif
endif
var nRow := SELF:_FindRowInBuffer(nRec)
@@ -1248,6 +1423,70 @@ partial class SQLRDD
SELF:_SetBOF(FALSE)
RETURN TRUE
+ ///
+ /// Replace the buffer with a small page that starts at record nRec, found by its key in the
+ /// current order (see _KeysetCondition()) instead of its offset: with OFFSET the server reads
+ /// through all rows before it (10 ms for a 10 row page at offset 500 of a 1000 row table, 2 ms
+ /// by key). Code that goes to a record mostly reads only it and the next or previous one;
+ /// reading on makes the pages grow, in both directions.
+ ///
+ /// The record's 0 based offset in the order when known, otherwise -1:
+ /// the buffer is then anchored at the record.
+ /// FALSE when the key cannot be expressed or the record is not part of the order.
+ PRIVATE METHOD _LoadWindowAt(nRec as DWORD, nOffset as long) AS LOGIC
+ local oRow as DataRow
+ if SELF:CurrentOrder != null
+ // only orders on plain columns, a key expression has no literal value
+ foreach var cItem in SELF:CurrentOrder:OrderList
+ var cExpr := iif(cItem:EndsWith(" DESC", StringComparison.OrdinalIgnoreCase), cItem:Substring(0, cItem:Length - 5), cItem)
+ if SELF:DataTable == null .or. SELF:DataTable:Columns:IndexOf(cExpr:Trim():Trim({'[',']','"','`'})) < 0
+ return false
+ endif
+ next
+ endif
+ var nRow := SELF:_FindRowInBuffer(nRec)
+ if nRow > 0
+ oRow := SELF:DataTable:Rows[nRow-1]
+ else
+ SELF:_command:CommandText := _builder:BuildDirectRecnoStatement(nRec)
+ SELF:_command:ClearParameters()
+ var oRecTable := SELF:_command:GetDataTable(SELF:Alias, SELF:_tableTemplate)
+ if oRecTable == null .or. oRecTable:Rows:Count == 0
+ return false
+ endif
+ oRow := oRecTable:Rows[0]
+ endif
+ var cFrom := SELF:_KeysetCondition(oRow, FALSE, TRUE, TRUE)
+ if cFrom == null
+ return false
+ endif
+ var nCount := SELF:_SmallPageRows
+ var newTable := SELF:_ReadTable(cFrom, 0, nCount)
+ if newTable == null .or. newTable:Rows:Count == 0 .or. Convert.ToUInt32(newTable:Rows[0][self:_recnoColumNo]) != nRec
+ // not part of the order (FOR condition, scope)
+ return false
+ endif
+ var nRows := newTable:Rows:Count
+ SELF:_lazyGoto := FALSE
+ SELF:_ClearTable()
+ SELF:DataTable := newTable
+ SELF:RowNumber := 1
+ SELF:_outsideOrder := false
+ SELF:_bufferContiguous := true
+ if nOffset >= 0
+ SELF:_anchorWhere := ""
+ SELF:_bufferStart := nOffset
+ else
+ SELF:_anchorWhere := cFrom
+ SELF:_bufferStart := 0
+ endif
+ SELF:_hasEOF := nRows < nCount
+ SELF:_readAheadRows := iif(nRows < nCount, 0, nCount * 2)
+ SELF:_readBackRows := nCount
+ SELF:_SetEOF(FALSE)
+ SELF:_SetBOF(FALSE)
+ return true
+
PRIVATE METHOD _GotoRecordOutsideOrder(nRec as DWORD) AS LOGIC
if !SELF:_LoadSingleRecord(nRec)
// Does not exist even physically.
@@ -1264,9 +1503,10 @@ partial class SQLRDD
///
PRIVATE METHOD _LoadSingleRecord(nRec as DWORD) AS LOGIC
try
+ SELF:_lazyGoto := FALSE
SELF:_command:CommandText := _builder:BuildDirectRecnoStatement(nRec)
SELF:_command:ClearParameters()
- var oTable := SELF:_command:GetDataTable(SELF:Alias)
+ var oTable := SELF:_command:GetDataTable(SELF:Alias, SELF:_tableTemplate)
if oTable == null .or. oTable:Rows:Count == 0
return false
endif
@@ -1294,14 +1534,58 @@ partial class SQLRDD
/// and under DBF/ADS the order switches are free. Positioning eagerly here meant a
/// ROW_NUMBER() over the whole order plus a page fetch at a large OFFSET for every switch.
///
- PRIVATE METHOD _RepositionAfterOrderChange(nRec as DWORD, lWasEof as LOGIC) AS VOID
- if SELF:_tableMode != TableMode.Table .or. SELF:_recnoColumNo == -1 .or. lWasEof
+ PRIVATE METHOD _RepositionAfterOrderChange(nRec as DWORD, lWasEof as LOGIC, aCurrentRow as object[]) AS VOID
+ if SELF:_tableMode != TableMode.Table .or. SELF:_recnoColumNo == -1
SELF:GoTo(iif(lWasEof, 0, nRec))
+ elseif lWasEof
+ // EOF stays EOF (e.g. switching back to the previous order after a seek that found
+ // nothing, before going back to the saved record). GoTo(0) would first load the
+ // first page of the new order; an empty buffer is enough to stand on the phantom
+ // record. _GoToPhantom() marks the EOF as pending (the empty buffer does not end at
+ // the last record), so Skip(1) stays at EOF and Skip(-1) goes to the bottom.
+ if SELF:_SetBufferRows(null) .or. SELF:_OpenTable("1=0", "")
+ SELF:_GoToPhantom(TRUE)
+ else
+ SELF:GoTo(0)
+ endif
+ elseif aCurrentRow != null .and. SELF:_SetBufferRows(aCurrentRow)
+ // The record we stand on was in the buffer a moment ago (written by GoCold() before the
+ // order change): keep it instead of reading it again. Its position in the new order is
+ // looked up when needed, as after _LoadSingleRecord().
+ SELF:_bufferContiguous := false
elseif !SELF:_LoadSingleRecord(nRec)
SELF:GoTo(nRec)
endif
return
+ ///
+ /// Replace the buffer with an empty table (aRow NULL) or one row, built from the structure of
+ /// the previous buffer without a query. FALSE when that structure is not known yet.
+ ///
+ PRIVATE METHOD _SetBufferRows(aRow as object[]) AS LOGIC
+ if SELF:_tableTemplate == null
+ return false
+ endif
+ var oTable := SELF:_tableTemplate:Clone()
+ if aRow != null
+ oTable:LoadDataRow(aRow, true)
+ endif
+ SELF:_lazyGoto := FALSE
+ SELF:_ClearTable()
+ SELF:DataTable := oTable
+ SELF:RowNumber := 1
+ SELF:_outsideOrder := false
+ SELF:_hasEOF := false
+ SELF:_anchorWhere := ""
+ SELF:_bufferStart := 0
+ SELF:_bufferContiguous := true
+ SELF:_readAheadRows := 0
+ SELF:_readBackRows := 0
+ SELF:_serverReccountValid := false
+ SELF:_SetEOF(oTable:Rows:Count == 0)
+ SELF:_SetBOF(FALSE)
+ return true
+
///
/// When the buffer only holds records whose position in the current order is unknown
/// (see _LoadSingleRecord()), look that position up for the current record now.
diff --git a/src/Runtime/XSharp.SQLRdd/Support/SqlDbTableCommandBuilder.prg b/src/Runtime/XSharp.SQLRdd/Support/SqlDbTableCommandBuilder.prg
index 9ef55a60fb..538b5fc227 100644
--- a/src/Runtime/XSharp.SQLRdd/Support/SqlDbTableCommandBuilder.prg
+++ b/src/Runtime/XSharp.SQLRdd/Support/SqlDbTableCommandBuilder.prg
@@ -150,7 +150,8 @@ internal class SqlDbTableCommandBuilder
/// Build a fetch of nCount rows of the current order/scope/filter (further restricted by
/// sWhereClause), starting at the 0-based row offset nOffset within that result.
///
- method BuildSqlStatement(sWhereClause as string, nOffset as int, nCount as int) as string
+ /// TRUE to sort in reverse, for reading the rows before a given one.
+ method BuildSqlStatement(sWhereClause as string, nOffset as int, nCount as int, lReverse := false as logic) as string
var sb := System.Text.StringBuilder{}
local scopeWhere := null as string
var CurrentOrder := _oRdd:CurrentOrder
@@ -197,9 +198,27 @@ internal class SqlDbTableCommandBuilder
cOrderby += " DESC"
endif
endif
+ if lReverse
+ // per item, the items can be expressions with commas
+ var aItems := List{}
+ foreach var cItem in CurrentOrder:OrderList
+ if cItem:EndsWith(" DESC", StringComparison.OrdinalIgnoreCase)
+ aItems:Add(cItem:Substring(0, cItem:Length - 5):TrimEnd())
+ else
+ aItems:Add(cItem + " DESC")
+ endif
+ next
+ if SELF:_oTable:HasRecnoColumn .and. !Functions.List2String(CurrentOrder:OrderList):Contains(Provider:QuoteIdentifier(self:_oTable:RecnoColumn))
+ aItems:Add(Provider:QuoteIdentifier(self:_oTable:RecnoColumn) + iif(CurrentOrder:Descending, "", " DESC"))
+ endif
+ cOrderby := Functions.List2String(aItems)
+ endif
elseif SELF:_oTable:HasRecnoColumn
sb:Append(Provider:OrderByClause)
cOrderby := Provider:QuoteIdentifier(self:_oTable:RecnoColumn)
+ if lReverse
+ cOrderby += " DESC"
+ endif
endif
cOrderby :=_connection:RaiseStringEvent(_connection, SqlRDDEventReason.OrderByClause, _cTable, cOrderby)
sb:Replace(SqlDbProvider.ColumnsMacro, cOrderby)
@@ -223,7 +242,7 @@ internal class SqlDbTableCommandBuilder
/// OFFSET 0 is always cheap regardless of table size; the caller is responsible for
/// reversing the returned rows back into ascending order.
///
- method BuildLastPageStatement() as string
+ method BuildLastPageStatement(nRows := 0 as long) as string
var sb := System.Text.StringBuilder{}
local scopeWhere := null as string
var CurrentOrder := _oRdd:CurrentOrder
@@ -277,7 +296,7 @@ internal class SqlDbTableCommandBuilder
sb:Replace(SqlDbProvider.ColumnsMacro, cOrderby)
sb:Append(Provider:PagingClause)
- sb:Replace(SqlDbProvider.PagesizeMacro, _oTable:PageSize:ToString())
+ sb:Replace(SqlDbProvider.PagesizeMacro, iif(nRows > 0, nRows, _oTable:PageSize):ToString())
sb:Replace(SqlDbProvider.StartRecMacro, "0")
return sb:ToString()