diff --git a/src/Runtime/XSharp.SQLRdd/Classes/Connection.prg b/src/Runtime/XSharp.SQLRdd/Classes/Connection.prg index 9dc87680fd..b60e0c401a 100644 --- a/src/Runtime/XSharp.SQLRdd/Classes/Connection.prg +++ b/src/Runtime/XSharp.SQLRdd/Classes/Connection.prg @@ -539,6 +539,70 @@ class SqlDbConnection inherit SqlDbHandleObject implements IDisposable return result end method + private _literalEncoding as System.Text.Encoding + private _literalEncodingRead as logic + /// + /// TRUE when cValue can be written as a string literal without a character being changed: ASCII, + /// or all characters in the code page of the database (see SqlDbProvider.CodePageStatement, read + /// once per connection). + /// + internal method IsExactStringLiteral(cValue as string) as logic + local lAscii := true as logic + foreach var c in cValue + if c > (char) 127 + lAscii := false + exit + endif + next + if lAscii + return true + endif + if !_literalEncodingRead + _literalEncodingRead := true + if SELF:Provider is SqlDbProvider var oProvider .and. !String.IsNullOrEmpty(oProvider:CodePageStatement) + try + var oResult := SELF:ExecuteScalar(oProvider:CodePageStatement) + var nCodePage := iif(oResult == null .or. oResult == DBNull.Value, 0, Convert.ToInt32(oResult)) + if nCodePage > 0 + _literalEncoding := System.Text.Encoding.GetEncoding(nCodePage, System.Text.EncoderFallback.ExceptionFallback, System.Text.DecoderFallback.ExceptionFallback) + endif + catch + _literalEncoding := null + end try + endif + endif + if _literalEncoding == null + return false + endif + try + return _literalEncoding:GetString(_literalEncoding:GetBytes(cValue)) == cValue + catch + return false + end try + end method + + private _tableRows := Dictionary{StringComparer.OrdinalIgnoreCase} as Dictionary + /// Approximate number of rows of a table from the statistics of the database (see + /// SqlDbProvider.TableRowsStatement), read once per connection; -1 when unknown. + internal method ApproximateRows(cTable as string) as int64 + local nRows as int64 + if String.IsNullOrEmpty(cTable) + return -1 + endif + if _tableRows:TryGetValue(cTable, out nRows) + return nRows + endif + nRows := -1 + if SELF:Provider is SqlDbProvider var oProvider .and. !String.IsNullOrEmpty(oProvider:TableRowsStatement) + var oResult := SELF:ExecuteScalar(oProvider:TableRowsStatement:Replace(SqlDbProvider.TableNameMacro, cTable:Replace("'", "''")), cTable) + if oResult != null .and. oResult != DBNull.Value + nRows := Convert.ToInt64(oResult) + endif + endif + _tableRows[cTable] := nRows + return nRows + end method + /// /// Execute a SQL command /// diff --git a/src/Runtime/XSharp.SQLRdd/DBMS/DbProvider.prg b/src/Runtime/XSharp.SQLRdd/DBMS/DbProvider.prg index 3c137838da..59c902b5bb 100644 --- a/src/Runtime/XSharp.SQLRdd/DBMS/DbProvider.prg +++ b/src/Runtime/XSharp.SQLRdd/DBMS/DbProvider.prg @@ -231,6 +231,40 @@ abstract class SqlDbProvider inherit SqlDbObject implements ISqlDbProvider return cPattern end method + /// + /// A date without a time of day as a SQL literal that compares exactly like the stored value, or + /// NULL when there is no such literal (the RDD then reads the value with a subquery). + /// The default implementation returns NULL: how a string converts to a date depends on the + /// database and the settings of the session. + /// + virtual method DateLiteral(dValue as DateTime) as string + return null + end method + + /// + /// Appended to a query of the RDD whose condition has OR, LIKE or range terms (the keyset condition + /// of a page, a soft seek, a scope), on a table with at least RangeQueryHintMinRows rows. The default + /// implementation returns an empty string. + /// + virtual property RangeQueryHint as string => "" + + /// + /// Statement that returns the code page in which the database stores the string literals of a + /// statement (characters outside it are changed). The default implementation returns an empty + /// string (unknown: the RDD only writes ASCII strings as literals). + /// + virtual property CodePageStatement as string => "" + + /// Minimum number of rows of a table (see TableRowsStatement) for RangeQueryHint. + virtual property RangeQueryHintMinRows as long => 100000 + + /// + /// Statement that returns the approximate number of rows of the table TableNameMacro from the + /// statistics of the database, without counting them. The default implementation returns an + /// empty string (unknown). + /// + virtual property TableRowsStatement as string => "" + /// /// Create a new SqlDbProvider object /// diff --git a/src/Runtime/XSharp.SQLRdd/DBMS/SqlServer.prg b/src/Runtime/XSharp.SQLRdd/DBMS/SqlServer.prg index 81c2a9150e..29f9407491 100644 --- a/src/Runtime/XSharp.SQLRdd/DBMS/SqlServer.prg +++ b/src/Runtime/XSharp.SQLRdd/DBMS/SqlServer.prg @@ -58,6 +58,38 @@ class SqlDbProviderSqlServer inherit SqlDbProvider /// SQL Server also reads [...] in a LIKE pattern as a character class. override property LikeWildcards as string => "%_[" + /// + /// 'yyyyMMdd' is read the same way for date, datetime, datetime2 and smalldatetime, + /// whatever the language and DATEFORMAT of the session. + override method DateLiteral(dValue as DateTime) as string + return "'" + dValue:ToString("yyyyMMdd", System.Globalization.CultureInfo.InvariantCulture) + "'" + end method + + /// + /// + /// A keyset condition k1 >= v1 AND (k1 > v1 OR (k1 = v1 AND k2 > v2) OR ...) can only be turned + /// into seeks on the index when the server knows the values. With PARAMETERIZATION FORCED (or a + /// plan from the cache) it does not: it seeks on k1 alone and reads all rows of v1 before the + /// position. Paging the lines of an order near the end of a table with 5.4 million rows read + /// 6,641 pages (860 ms CPU) for 10 rows, with OPTION (RECOMPILE) 54 pages (2 ms to compile). + /// A range is at risk as well: a plan from the cache for TYP = @0 AND KURZNAME >= @1 ORDER BY + /// KURZNAME, made for the 1.1 million customers (TYP 'K'), read the short name index in order and + /// looked up TYP for every row. For a soft seek among the 488 suppliers (TYP 'L') near the end of + /// the names it read 720,398 pages (107 s) to find nothing; recompiled, 418 pages (6 ms). + /// On small tables these reads are cheap and compiling every query costs more: navigation tests on + /// tables with 1,000 to 50,000 rows took 1.5 to 2 times as long, hence RangeQueryHintMinRows. + /// + override property RangeQueryHint as string => " OPTION (RECOMPILE)" + + /// + /// A varchar literal takes the default collation of the database. + override property CodePageStatement as string => ; + "SELECT COLLATIONPROPERTY(CONVERT(sysname, DATABASEPROPERTYEX(DB_NAME(), 'Collation')), 'CodePage')" + + /// + override property TableRowsStatement as string => ; + "SELECT SUM(rows) FROM sys.partitions WHERE object_id = OBJECT_ID(N'" + TableNameMacro + "') AND index_id IN (0, 1)" + private static aFuncs := null as Dictionary /// override method GetFunctions() as Dictionary diff --git a/src/Runtime/XSharp.SQLRdd/RDD/SQLDbOrder.prg b/src/Runtime/XSharp.SQLRdd/RDD/SQLDbOrder.prg index 75d4523a38..66b9364685 100644 --- a/src/Runtime/XSharp.SQLRdd/RDD/SQLDbOrder.prg +++ b/src/Runtime/XSharp.SQLRdd/RDD/SQLDbOrder.prg @@ -318,6 +318,8 @@ internal class SqlDbOrder inherit SqlDbObject foreach var oTerm in terms if oTerm:IsDate widths:Add(8) + elseif oTerm:Width > 0 + widths:Add((int) oTerm:Width) else var cName := oTerm:Column:Trim({'[',']','"','`'}) var oCol := self:RDD:TableColumns:FirstOrDefault({ c => String.Compare(c:Name, cName, true) == 0 }) @@ -361,7 +363,7 @@ internal class SqlDbOrder inherit SqlDbObject sb:Append(SqlDbProvider.AndClause) endif if parts[i]:Length >= widths[i] - sb:Append(columns[i] + " = " + Functions.XsValueToSqlValue(parts[i])) + sb:Append(SELF:_PartCompare(terms[i], columns[i], " = ", parts[i])) else sb:Append(columns[i] + " like " + Provider:LikePattern(parts[i])) endif @@ -379,17 +381,26 @@ internal class SqlDbOrder inherit SqlDbObject endif sb:Append("(") for var j := 0 upto i-1 - sb:Append(columns[j] + " = " + Functions.XsValueToSqlValue(parts[j])) + sb:Append(SELF:_PartCompare(terms[j], columns[j], " = ", parts[j])) sb:Append(SqlDbProvider.AndClause) next - var cColExpr := columns[i] + var cComp := iif(i == parts:Count-1, cRangeComp, cStrictComp) if parts[i]:Length < widths[i] - // Compare only the prefix of the column that the value actually covers. - var cSubstr := Provider:GetFunction("SUBSTR(%1%,%2%,%3%)") - cColExpr := cSubstr:Replace("%1%", cColExpr):Replace("%2%","1"):Replace("%3%", parts[i]:Length:ToString()) + // Compare only the prefix of the column that the value actually covers (only + // the last part can be shorter, so this is the inclusive bound). SUBSTRING(c,1,n) + // would keep the server from seeking on the index: a soft seek in a descending + // order of 3.8 million orders took 0.7 to 1.7 seconds. The same conditions: + // SUBSTRING(c,1,n) >= v is c >= v + // SUBSTRING(c,1,n) <= v is c < v OR c LIKE 'v%' + if cRangeComp:Contains(">") + sb:Append(columns[i] + " >= " + Functions.XsValueToSqlValue(parts[i])) + else + sb:Append("(" + columns[i] + " < " + Functions.XsValueToSqlValue(parts[i]) + ; + SqlDbProvider.OrClause + columns[i] + " like " + Provider:LikePattern(parts[i]) + ")") + endif + else + sb:Append(SELF:_PartCompare(terms[i], columns[i], cComp, parts[i])) endif - var cComp := iif(i == parts:Count-1, cRangeComp, cStrictComp) - sb:Append(cColExpr + cComp + Functions.XsValueToSqlValue(parts[i])) sb:Append(")") next if parts:Count > 1 @@ -400,5 +411,19 @@ internal class SqlDbOrder inherit SqlDbObject return sb:ToString() end method + /// + /// cExpr cComp cValue for a key part the value covers completely. A STR() part compares the number, + /// it is sorted by the number (see SqlDbExpression.GetKeyTerms()): [MENGE] = 1 instead of + /// STR([MENGE],5,0) = ' 1', which also lets the server seek on an index with MENGE. + /// + private method _PartCompare(oTerm as SqlDbKeyTerm, cExpr as string, cComp as string, cValue as string) as string + local nValue as decimal + if oTerm:Width > 0 .and. Decimal.TryParse(cValue:Trim(), System.Globalization.NumberStyles.AllowLeadingSign | System.Globalization.NumberStyles.AllowDecimalPoint, ; + System.Globalization.CultureInfo.InvariantCulture, out nValue) + return oTerm:Column + cComp + nValue:ToString(System.Globalization.CultureInfo.InvariantCulture) + endif + return cExpr + cComp + Functions.XsValueToSqlValue(cValue) + end method + end class end namespace // XSharp.RDD.SqlRDD diff --git a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg index fc2d5a24af..16ee07f3b0 100644 --- a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg +++ b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Main.prg @@ -702,25 +702,17 @@ partial class SQLRDD inherit Workarea SELF:_outsideOrder := FALSE SELF:_eofPending := FALSE SELF:_ClearTable() - // Route through the cached count instead of always querying directly - GoBottom() can - // be called many times in a row against unchanged data (e.g. Lister construction - // retrying GoTo(0) when it can't establish a valid position), and each direct query - // here used to pay for a fresh COUNT(*) every single time. - self:_GetRecCount() - local nTotal := (long) self:_serverReccount as long SELF:_anchorWhere := "" - // For a large table, fetching the last page via the normal ascending, OFFSET-based - // query forces the server to walk/skip almost the entire ordered result - - // the further from the top, the worse. _FetchLastPage avoids that by sorting in - // reverse and asking for OFFSET 0 instead, which is always cheap. Only safe when the - // order isn't already descending (that would invert the reversal); falls back to the - // original approach for natural order, descending orders, or if it fails outright. - if self:CurrentOrder == null .or. !self:CurrentOrder:Descending - if !SELF:_FetchLastPage(nTotal) - SELF:_LoadBufferAt(SELF:_LastPageStart(nTotal)) - endif - else - SELF:_LoadBufferAt(SELF:_LastPageStart(nTotal)) + // For a large table, fetching the last page via the normal OFFSET-based query forces + // the server to walk/skip almost the entire ordered result - the further from the top, + // the worse: OFFSET 3768900 in a descending order of 3.8 million orders did not finish + // within 30 seconds. _FetchLastPage avoids that by sorting in reverse and asking for + // OFFSET 0 instead, which is always cheap. Falls back to the OFFSET only if it fails. + // The count is only needed for the OFFSET (see _FetchLastPage()); it goes through the + // cached count - GoBottom() can be called many times in a row against unchanged data. + if !SELF:_FetchLastPage() + self:_GetRecCount() + SELF:_LoadBufferAt(SELF:_LastPageStart((long) self:_serverReccount)) endif SELF:RowNumber := SELF:RowCount SELF:_Top := FALSE diff --git a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Orders.prg b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Orders.prg index af0b8a4875..cab5179056 100644 --- a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Orders.prg +++ b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Orders.prg @@ -566,6 +566,9 @@ partial class SQLRDD return false endif var cSeekWhere := CurrentOrder:SeekExpression(seekInfo ) + if seekInfo:Last .and. SELF:_SeekLastDirect(seekInfo) + return true + endif // The buffer is anchored at the soft-seek condition for the same value: the rows a // hard seek returns are exactly the leading rows of that result, so paging forward past // them (the common "seek to the first record of a key, then Skip() while the key still @@ -652,6 +655,42 @@ partial class SQLRDD return true end method + /// + /// SeekLast: read the last record of the key directly, with the order reversed, and position + /// on it. FALSE when that is not possible (filter, no recno column) or the key has no record: + /// the caller then continues as before (soft seek, not found). + /// + /// + /// Seek() found the first record of the key and skipped from there to the last one. That reads + /// every record of the key, page by page: BelegHdr:GoTop() in our ERP seeks the newest order of + /// a type with SeekLast on BEL_ART+DTOS(DAT_HINZU)+BEL_NR, and with 3.8 million orders it had + /// not arrived after 80 seconds and 310,000 records. The position of the record in the order is + /// looked up when something moves from it, as after GoTo(). + /// + PRIVATE METHOD _SeekLastDirect(seekInfo as DbSeekInfo) AS LOGIC + if SELF:_FilterInfo:Active .or. SELF:_tableMode != TableMode.Table .or. SELF:_recnoColumNo < 0 + return false + endif + // the records of the key also for a soft seek: it only differs when the key has none + var hardInfo := seekInfo + hardInfo:SoftSeek := FALSE + hardInfo:Last := FALSE + var oLast := SELF:_ReadTable(SELF:CurrentOrder:SeekExpression(hardInfo), 0, 1, TRUE) + if oLast == null .or. oLast:Rows:Count == 0 + SELF:_ReturnSpareTable(oLast) + return false + endif + var nRec := Convert.ToUInt32(oLast:Rows[0][SELF:_recnoColumNo]) + SELF:_ReturnSpareTable(oLast) + if !SELF:GoTo(nRec) .or. SELF:_EoF + return false + endif + SELF:_SetEOF(FALSE) + SELF:_SetBOF(FALSE) + SELF:Found := TRUE + return true + end method + PRIVATE METHOD KeyCompare(oRecKey as OBJECT, oKey as OBJECT) AS LONG LOCAL result AS INT64 IF oRecKey IS NULL diff --git a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg index a08a06534d..59bab9afc3 100644 --- a/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg +++ b/src/Runtime/XSharp.SQLRdd/RDD/SQLRDD-Private.prg @@ -139,6 +139,19 @@ partial class SQLRDD return _connection?:RecnoColumn end get end property + /// Length of the character column cName, -1 when it is no character column or unknown. + /// Also while the production index is parsed: the fields of the work area are not there yet. + internal method _CharColumnLength(cName as string) as long + var oTable := _oTd ?? _builder?:TableInfo + if oTable?:Columns != null + foreach var oCol in oTable:Columns + if String.Equals(oCol:Name, cName, StringComparison.OrdinalIgnoreCase) + return iif(oCol:ColumnInfo:FieldType == DbFieldType.Character, oCol:Length, -1) + endif + next + endif + return -1 + end method /// Maximum number of rows kept in the local buffer. private property _MaxBufferRows as long => _oTd:PageSize * Math.Max(_oTd:BufferSize, 1) /// Rows for a first read where only a few records are expected to be needed (after a seek, @@ -646,10 +659,22 @@ partial class SQLRDD /// Fetch the last page of the current order/scope/filter directly, without going through /// the ascending, huge-OFFSET query GoBottom() would otherwise need for a large table. /// - /// See SqlDbTableCommandBuilder.BuildLastPageStatement for why this exists. - private method _FetchLastPage(nTotalRows as long) as logic + /// + /// See SqlDbTableCommandBuilder.BuildLastPageStatement for why this exists. + /// The position of the page in the order is only known from a count of the order. When no valid + /// count is cached, the page is anchored at its first row instead (as after GoTo(), see + /// _anchorWhere), and the position is looked up when something needs it. A list of the lines of + /// an order in our ERP calls GoBottom() on every move to another order: each one counted the + /// order of a table with 5.4 million rows first (46 ms), 72 times while paging through orders. + /// + private method _FetchLastPage() as logic try - SELF:_command:CommandText := _builder:BuildLastPageStatement(SELF:_SmallPageRows) + // A descending order is reversed item by item, like the backward keyset reads do. + if SELF:CurrentOrder != null .and. SELF:CurrentOrder:Descending + SELF:_command:CommandText := SELF:_BuildSqlStatement("", 0, SELF:_SmallPageRows, TRUE) + else + SELF:_command:CommandText := _builder:BuildLastPageStatement(SELF:_SmallPageRows) + endif SELF:_command:ClearParameters() var newTable := SELF:_command:GetDataTable(SELF:Alias, SELF:_tableTemplate, SELF:_TakeSpareTable()) if newTable == null @@ -668,7 +693,21 @@ partial class SQLRDD SELF:_ReturnSpareTable(newTable) SELF:_anchorWhere := "" SELF:_bufferContiguous := true - SELF:_bufferStart := Math.Max(0, nTotalRows - SELF:RowCount) + if SELF:_serverReccountValid + SELF:_bufferStart := Math.Max(0, (long) SELF:_serverReccount - SELF:RowCount) + elseif SELF:RowCount < SELF:_SmallPageRows + // fewer rows than asked for: the whole order + SELF:_bufferStart := 0 + else + var cAnchor := SELF:_KeysetCondition(SELF:DataTable:Rows[0], FALSE, TRUE, TRUE) + if cAnchor != null + SELF:_anchorWhere := cAnchor + SELF:_bufferStart := 0 + else + SELF:_GetRecCount() + SELF:_bufferStart := Math.Max(0, (long) SELF:_serverReccount - SELF:RowCount) + endif + endif // reading backwards from the bottom starts small as well SELF:_readBackRows := SELF:_SmallPageRows // This IS the last page by definition - without this, a Skip(1) right after @@ -996,8 +1035,10 @@ partial class SQLRDD /// Every subquery has to be optimized by the server: with one per item, compiling the keyset /// condition of a two column key took 30-45 ms, executing it 3 ms. Since the text changes with /// every page the plan is never reused. The buffer holds the values as read from the server, so - /// a literal is exact for strings (ASCII only, so no code page conversion can change them) and - /// integers/decimals. + /// a literal is exact for strings (ASCII only, so no code page conversion can change them), + /// integers/decimals, and dates without a time of day when the provider has a literal for them + /// (SqlDbProvider.DateLiteral()). With a subquery for a date, SQL Server could not seek on the + /// index: every page of a 3.8 million row table read all of it (665,762 pages against 1,941). /// private method _KeysetLiteral(oRow as DataRow, cExpr as string) as string var nCol := SELF:DataTable:Columns:IndexOf(cExpr:Trim({'[',']','"','`'})) @@ -1006,11 +1047,12 @@ partial class SQLRDD endif switch oRow[nCol] case cValue as string - foreach var c in cValue - if c > (char) 127 .or. c == (char) 92 - return null - endif - next + // Non-ASCII characters only when the database stores them unchanged in a literal: a key + // like 'GÜNTHER' fell back to the position of the record, an OFFSET of 323,815 rows in + // the list of 1.1 million addresses (more than a minute). + if cValue:IndexOf((char) 92) >= 0 .or. !SELF:Connection:IsExactStringLiteral(cValue) + return null + endif return Functions.XsValueToSqlValue(cValue) case nValue as long return nValue:ToString(System.Globalization.CultureInfo.InvariantCulture) @@ -1020,6 +1062,11 @@ partial class SQLRDD return nValue:ToString(System.Globalization.CultureInfo.InvariantCulture) case nValue as decimal return nValue:ToString(System.Globalization.CultureInfo.InvariantCulture) + case dValue as DateTime + // a time of day would need the precision of the column type + if dValue:TimeOfDay == TimeSpan.Zero .and. SELF:Provider is SqlDbProvider var oProvider + return oProvider:DateLiteral(dValue) + endif end switch return null end method @@ -1270,8 +1317,11 @@ partial class SQLRDD SELF:_SetBOF(TRUE) return true endif - if !String.IsNullOrEmpty(SELF:_anchorWhere) .and. 1 - nNewRow <= _oTd:PageSize + if !String.IsNullOrEmpty(SELF:_anchorWhere) .and. 1 - nNewRow <= SELF:_MaxBufferRows // Read the rows before an anchored buffer by key, without looking up its position. + // Also for more than a page: the position of a buffer near the end of a large order + // is far down, and the page by position needs an OFFSET of millions of rows (two + // minutes for 150 rows back from the end of 3.8 million orders). var nFetched := SELF:_FetchBackwardKeyset(1 - nNewRow) if nFetched == 0 SELF:RowNumber := 1 @@ -1305,7 +1355,8 @@ partial class SQLRDD return true endif nTarget := SELF:_bufferStart + nNewRow - 1 - if 1 - nNewRow > _oTd:PageSize + // up to what the buffer holds _FetchBackward() reads by key (see above), beyond that by position + if 1 - nNewRow > SELF:_MaxBufferRows SELF:_LoadBufferAt(nTarget) if nTarget < 0 SELF:RowNumber := 1 diff --git a/src/Runtime/XSharp.SQLRdd/Support/Expression.prg b/src/Runtime/XSharp.SQLRdd/Support/Expression.prg index ab2af4f656..49be8e9e0d 100644 --- a/src/Runtime/XSharp.SQLRdd/Support/Expression.prg +++ b/src/Runtime/XSharp.SQLRdd/Support/Expression.prg @@ -294,6 +294,7 @@ internal class SqlDbExpression inherit SqlDbObject // fixed-width columns, so sorting by the raw columns gives the same order as sorting // by the concatenated text - and the server can read it from an index on // (REF_NR, DATUM) instead of computing and sorting the text for every row. + // ART_NR+LISTE+STR(MENGE,5,0) the same way, see GetKeyTerms() for the STR() part. aList := List{} foreach var oTerm in self:KeyTerms aList:Add(oTerm:Column) @@ -450,8 +451,18 @@ internal class SqlDbExpression inherit SqlDbObject return access SegmentCount as long return self:Segments:Count + /// TRUE when oField is a character column of the table that cLiteral covers completely. + private method _CoversField(oField as SqlDbToken, cLiteral as string) as logic + if oField == null .or. oField:Type != TokenType.Token .or. SELF:Owner == null .or. SELF:Owner:RDD == null + return false + endif + var nLen := SELF:Owner:RDD:_CharColumnLength(oField:Name) + return nLen > 0 .and. cLiteral:Length >= nLen + end method + method TranslateFunctions() as void local lastToken := null as SqlDbToken + local prevToken := null as SqlDbToken foreach var token in self:Tokens switch token:Type case TokenType.BoFunc @@ -475,15 +486,21 @@ internal class SqlDbExpression inherit SqlDbObject case TokenType.Token token:SQLName := _provider.QuoteIdentifier(token:Name) case TokenType.String - if lastToken != null .and. lastToken:Name:Trim() == "=" + if lastToken != null .and. lastToken:Name:Trim() == "=" .and. ; + !SELF:_CoversField(prevToken, token:Name:Substring(1, token:Name:Length-2)) // A single equals compares like a prefix in xBase (SET EXACT OFF): field = 'A' // is TRUE for 'AB'. The literal (without its quotes) is matched literally. lastToken:SQLName := "like" token:SQLName := _provider:LikePattern(token:Name:Substring(1, token:Name:Length-2)) else + // A literal as long as the field compares the whole field: the prefix compare + // is an equality. LIKE is a range for the server: with TYP = 'L' (TYP C(1)) as + // FOR condition of ART_NR+REF_NR, it sorted all 477,757 rows of type L for the + // first page instead of reading them in the order of the index. token:SQLName := token:Name endif end switch + prevToken := lastToken lastToken := token next self:Translated := true @@ -528,7 +545,8 @@ internal class SqlDbExpression inherit SqlDbObject return cK /// - /// Split a key into its parts when it only concatenates columns and DTOS(column), else NULL. + /// Split a key into its parts when it only concatenates columns, DTOS(column) and + /// STR(column, n[, d]), else NULL. /// private method GetKeyTerms(cKey as string) as List var aTerms := List{} @@ -546,6 +564,24 @@ internal class SqlDbExpression inherit SqlDbObject endif var cCol := _provider:QuoteIdentifier(cName) aTerms:Add(SqlDbKeyTerm{cCol, cDtos:Replace("%1%", cCol), true}) + elseif cTerm:StartsWith("STR(", StringComparison.OrdinalIgnoreCase) .and. cTerm:EndsWith(")") + // STR(MENGE,5,0): a part of fixed width n, sorted by the number. Only then an index on + // (ART_NR, LISTE, MENGE) gives the order: sorted by the text, the server sorted all + // rows behind a position before it could return the next page (118,278 reads against + // 43). The text sorts like the number for values from 0 to the width; unlike DBF, + // negative values come before 0 and values wider than n ('*****') at the end. + var aArgs := cTerm:Substring(4, cTerm:Length - 5):Split({c','}) + var cName := aArgs[0]:Trim() + local nLen := 10 as long + local nDec := 0 as long + if aArgs:Length > 3 .or. !IsValidToken(cName) .or. cName:Length == 0 ; + .or. (aArgs:Length > 1 .and. !Int32.TryParse(aArgs[1]:Trim(), out nLen)) ; + .or. (aArgs:Length > 2 .and. !Int32.TryParse(aArgs[2]:Trim(), out nDec)) .or. nLen <= 0 .or. nDec < 0 + return null + endif + var cCol := _provider:QuoteIdentifier(cName) + var cStr := _provider:GetFunction("STR(%1%,%2%,%3%)"):Replace("%1%", cCol):Replace("%2%", nLen:ToString()):Replace("%3%", nDec:ToString()) + aTerms:Add(SqlDbKeyTerm{cCol, cStr, false, nLen}) else return null endif @@ -562,10 +598,13 @@ internal class SqlDbKeyTerm property SqlExpr as string auto get private set /// TRUE for DTOS(column): always 8 characters (yyyymmdd) property IsDate as logic auto get private set - constructor(cColumn as string, cSqlExpr as string, lDate as logic) + /// Width of a STR(column, n) part, 0 for the other parts + property Width as long auto get private set + constructor(cColumn as string, cSqlExpr as string, lDate as logic, nWidth := 0 as long) self:Column := cColumn self:SqlExpr := cSqlExpr self:IsDate := lDate + self:Width := nWidth return end class end namespace // XSharp.RDD.SqlRDD diff --git a/src/Runtime/XSharp.SQLRdd/Support/SqlDbTableCommandBuilder.prg b/src/Runtime/XSharp.SQLRdd/Support/SqlDbTableCommandBuilder.prg index 7790a61749..79cff650bd 100644 --- a/src/Runtime/XSharp.SQLRdd/Support/SqlDbTableCommandBuilder.prg +++ b/src/Runtime/XSharp.SQLRdd/Support/SqlDbTableCommandBuilder.prg @@ -226,9 +226,28 @@ internal class SqlDbTableCommandBuilder sb:Append(Provider:PagingClause) sb:Replace(SqlDbProvider.PagesizeMacro, nCount:ToString()) sb:Replace(SqlDbProvider.StartRecMacro, nOffset:ToString()) + sb:Append(SELF:_QueryHint(sWhereClause)) return sb:ToString() + private static _rangeCondition := System.Text.RegularExpressions.Regex{"\s(or|like|<|>|<=|>=)\s", ; + System.Text.RegularExpressions.RegexOptions.IgnoreCase} as System.Text.RegularExpressions.Regex + + /// + /// The provider's hint for a query with this condition (see SqlDbProvider.RangeQueryHint): when it has + /// OR, LIKE or range terms (a keyset condition, a soft seek, a scope, a prefix seek) and the table is + /// large. A condition of equalities only keeps the plan from the cache, it is a seek for any value. + /// + private method _QueryHint(sWhereClause as string) as string + if String.IsNullOrEmpty(sWhereClause) .or. !_rangeCondition:IsMatch(sWhereClause) + return "" + endif + if Provider is SqlDbProvider var oProvider .and. !String.IsNullOrEmpty(oProvider:RangeQueryHint) ; + .and. _connection:ApproximateRows(self:_oTable:RealName) >= oProvider:RangeQueryHintMinRows + return oProvider:RangeQueryHint + endif + return "" + /// /// Build a fetch of the last page of the current order/scope/filter, without the classic /// "large OFFSET" performance problem. @@ -298,6 +317,7 @@ internal class SqlDbTableCommandBuilder sb:Append(Provider:PagingClause) sb:Replace(SqlDbProvider.PagesizeMacro, iif(nRows > 0, nRows, _oTable:PageSize):ToString()) sb:Replace(SqlDbProvider.StartRecMacro, "0") + sb:Append(SELF:_QueryHint(sWhereClause)) return sb:ToString()