From 255d2d33abf4c2c5a918bacde049004e52b7986f Mon Sep 17 00:00:00 2001 From: Neil Date: Tue, 17 Apr 2018 08:23:18 +1000 Subject: Add SC_DOCUMENTOPTION_TEXT_LARGE option for documents larger than 2 GigaBytes. This option is provisional and experimental. --- doc/ScintillaDoc.html | 49 +++++++++++++++++++++++++++++--------- doc/ScintillaHistory.html | 4 ++++ include/Scintilla.h | 4 +++- include/Scintilla.iface | 10 +++++--- src/CellBuffer.cxx | 17 +++++++++++-- src/CellBuffer.h | 5 +++- src/ContractionState.cxx | 7 ++++-- src/ContractionState.h | 2 +- src/Document.cxx | 9 +++++-- src/Document.h | 2 ++ src/EditModel.cxx | 2 +- src/EditView.cxx | 7 +++--- src/Editor.cxx | 9 ++++--- src/MarginView.cxx | 13 ++++++---- src/Position.h | 4 ++-- src/RunStyles.cxx | 2 ++ test/unit/testCellBuffer.cxx | 2 +- test/unit/testContractionState.cxx | 2 +- test/unit/testPartitioning.cxx | 2 +- 19 files changed, 111 insertions(+), 41 deletions(-) diff --git a/doc/ScintillaDoc.html b/doc/ScintillaDoc.html index 25e7ca15b..d9f658814 100644 --- a/doc/ScintillaDoc.html +++ b/doc/ScintillaDoc.html @@ -119,7 +119,7 @@

Scintilla Documentation

-

Last edited 1 February 2018 NH

+

Last edited 17 April 2018 NH

There is an overview of the internal design of Scintilla.
@@ -5711,11 +5711,12 @@ sptr_t CallScintilla(unsigned int iMessage, uptr_t wParam, sptr_t lParam){ SCI_GETDOCPOINTER → document *
SCI_SETDOCPOINTER(<unused>, document *doc)
- SCI_CREATEDOCUMENT(int bytes, int documentOption) → document *
+ SCI_CREATEDOCUMENT(int bytes, int documentOptions) → document *
SCI_ADDREFDOCUMENT(<unused>, document *doc)
SCI_RELEASEDOCUMENT(<unused>, document *doc)
+ SCI_GETDOCUMENTOPTIONS → int

SCI_GETDOCPOINTER → document *
@@ -5732,7 +5733,7 @@ sptr_t CallScintilla(unsigned int iMessage, uptr_t wParam, sptr_t lParam){ window.
6. If doc was not 0, its reference count is increased by 1.

-

SCI_CREATEDOCUMENT(int bytes, int documentOption) → document *
+

SCI_CREATEDOCUMENT(int bytes, int documentOptions) → document *
This message creates a new, empty document and returns a pointer to it. This document is not selected into the editor and starts with a reference count of 1. This means that you have ownership of it and must either reduce its reference count by 1 after using @@ -5743,12 +5744,29 @@ sptr_t CallScintilla(unsigned int iMessage, uptr_t wParam, sptr_t lParam){ to allocate once rather than rely on the buffer growing as data is added. If SCI_CREATEDOCUMENT fails then 0 is returned.

-

The documentOption argument +

The documentOptions argument chooses between different document capabilities which affect memory allocation and performance with SC_DOCUMENTOPTION_DEFAULT (0) choosing standard options. - SC_DOCUMENTOPTION_STYLES_NONE (1) stops allocation of memory to style characters + SC_DOCUMENTOPTION_STYLES_NONE (0x1) stops allocation of memory to style characters which saves significant memory, often 40% with the whole document treated as being style 0. - Lexers may still produce visual styling by using indicators.

+ Lexers may still produce visual styling by using indicators. + SC_DOCUMENTOPTION_TEXT_LARGE (0x100) accomodates documents larger than 2 GigaBytes + in 64-bit executables. +

+ +

With SC_DOCUMENTOPTION_STYLES_NONE, lexers are still active and may display + indicators. Some may produce folding information althoough most require lexical styles to correctly determine folding. + Its often more efficient to set the null lexer SCLEX_NULL so no lexer is run. +

+ +

The SC_DOCUMENTOPTION_TEXT_LARGE option is experimental and has not been + thoroughly tested. Lexers may fail or hang when lexing past 2GB or 4GB. + Applications using this option should be tested to ensure the option works in their circumstances and each lexer + included should also be tested with documents larger than 4GB. + For many applications lexing documents larger than 4GB will be too sluggish so SC_DOCUMENTOPTION_STYLES_NONE + and the null lexer SCLEX_NULL can be used. Another approach is to turn on idle styling with + SCI_SETIDLESTYLING. +

@@ -5769,10 +5787,16 @@ sptr_t CallScintilla(unsigned int iMessage, uptr_t wParam, sptr_t lParam){ - + + + + + + +
SC_DOCUMENTOPTION_STYLES_NONE10x1 Stop allocation of memory for styles and treat all text as style 0.
SC_DOCUMENTOPTION_TEXT_LARGE0x100Allow document to be larger than 2 GB.
@@ -5794,6 +5818,9 @@ sptr_t CallScintilla(unsigned int iMessage, uptr_t wParam, sptr_t lParam){ world spinning in its orbit you must balance each call to SCI_CREATEDOCUMENT or SCI_ADDREFDOCUMENT with a call to SCI_RELEASEDOCUMENT.

+

SCI_GETDOCUMENTOPTIONS → int
+ Returns the options that were used to create the document.

+

Background loading and saving

To ensure a responsive user interface, applications may decide to load and save documents using a separate thread @@ -5801,7 +5828,7 @@ sptr_t CallScintilla(unsigned int iMessage, uptr_t wParam, sptr_t lParam){

Loading in the background

- SCI_CREATELOADER(int bytes, int documentOption) → int
+ SCI_CREATELOADER(int bytes, int documentOptions) → int

An application can load all of a file into a buffer it allocates on a background thread and then add the data in that buffer @@ -5810,15 +5837,15 @@ sptr_t CallScintilla(unsigned int iMessage, uptr_t wParam, sptr_t lParam){

To avoid these issues, a loader object may be created and used to load the file. The loader object supports the ILoader interface.

-

SCI_CREATELOADER(int bytes, int documentOption) → int
+

SCI_CREATELOADER(int bytes, int documentOptions) → int
Create an object that supports the ILoader interface which can be used to load data and then be turned into a Scintilla document object for attachment to a view object. The bytes argument determines the initial memory allocation for the document as it is more efficient to allocate once rather than rely on the buffer growing as data is added. If SCI_CREATELOADER fails then 0 is returned.

-

The documentOption argument - is described in the SCI_CREATEDOCUMENT section.

+

The documentOptions argument + is described in the SCI_CREATEDOCUMENT section.

ILoader

diff --git a/doc/ScintillaHistory.html b/doc/ScintillaHistory.html index 87bfa49ab..00fe782fe 100644 --- a/doc/ScintillaHistory.html +++ b/doc/ScintillaHistory.html @@ -542,6 +542,10 @@ Released 10 April 2018.
  • + Add experimental SC_DOCUMENTOPTION_TEXT_LARGE option to accomodate documents larger than + 2 GigaBytes. +
    • +
  • Set the last X chosen when SCI_REPLACESEL called to ensure macros work when text insertion followed by caret up or down.
    • diff --git a/include/Scintilla.h b/include/Scintilla.h index 29a723e26..3ec70a380 100644 --- a/include/Scintilla.h +++ b/include/Scintilla.h @@ -685,10 +685,12 @@ typedef sptr_t (*SciFnDirect)(sptr_t ptr, unsigned int iMessage, uptr_t wParam, #define SCI_SETZOOM 2373 #define SCI_GETZOOM 2374 #define SC_DOCUMENTOPTION_DEFAULT 0 -#define SC_DOCUMENTOPTION_STYLES_NONE 1 +#define SC_DOCUMENTOPTION_STYLES_NONE 0x1 +#define SC_DOCUMENTOPTION_TEXT_LARGE 0x100 #define SCI_CREATEDOCUMENT 2375 #define SCI_ADDREFDOCUMENT 2376 #define SCI_RELEASEDOCUMENT 2377 +#define SCI_GETDOCUMENTOPTIONS 2379 #define SCI_GETMODEVENTMASK 2378 #define SCI_SETFOCUS 2380 #define SCI_GETFOCUS 2381 diff --git a/include/Scintilla.iface b/include/Scintilla.iface index 51c3f2e81..2c361b5f0 100644 --- a/include/Scintilla.iface +++ b/include/Scintilla.iface @@ -1773,16 +1773,20 @@ get int GetZoom=2374(,) enu DocumentOption=SC_DOCUMENTOPTION_ val SC_DOCUMENTOPTION_DEFAULT=0 -val SC_DOCUMENTOPTION_STYLES_NONE=1 +val SC_DOCUMENTOPTION_STYLES_NONE=0x1 +val SC_DOCUMENTOPTION_TEXT_LARGE=0x100 # Create a new document object. # Starts with reference count of 1 and not selected into editor. -fun int CreateDocument=2375(int bytes, int documentOption) +fun int CreateDocument=2375(int bytes, int documentOptions) # Extend life of document. fun void AddRefDocument=2376(, int doc) # Release a reference to the document, deleting document if it fades to black. fun void ReleaseDocument=2377(, int doc) +# Get which document options are set. +get int GetDocumentOptions=2379(,) + # Get which document modification events are sent to the container. get int GetModEventMask=2378(,) @@ -2544,7 +2548,7 @@ set void SetTechnology=2630(int technology,) get int GetTechnology=2631(,) # Create an ILoader*. -fun int CreateLoader=2632(int bytes, int documentOption) +fun int CreateLoader=2632(int bytes, int documentOptions) # On OS X, show a find indicator. fun void FindIndicatorShow=2640(position start, position end) diff --git a/src/CellBuffer.cxx b/src/CellBuffer.cxx index d6d83c20b..fcc75a741 100644 --- a/src/CellBuffer.cxx +++ b/src/CellBuffer.cxx @@ -364,12 +364,17 @@ void UndoHistory::CompletedRedoStep() { currentAction++; } -CellBuffer::CellBuffer(bool hasStyles_) : - hasStyles(hasStyles_) { +CellBuffer::CellBuffer(bool hasStyles_, bool largeDocument_) : + hasStyles(hasStyles_), largeDocument(largeDocument_) { readOnly = false; utf8LineEnds = 0; collectingUndo = true; plv = std::make_unique>(); + if (largeDocument) + plv = std::make_unique>(); + else + plv = std::make_unique>(); + } CellBuffer::~CellBuffer() { @@ -556,6 +561,14 @@ void CellBuffer::SetReadOnly(bool set) { readOnly = set; } +bool CellBuffer::IsLarge() const { + return largeDocument; +} + +bool CellBuffer::HasStyles() const { + return hasStyles; +} + void CellBuffer::SetSavePoint() { uh.SetSavePoint(); } diff --git a/src/CellBuffer.h b/src/CellBuffer.h index 544a26711..935ea6e69 100644 --- a/src/CellBuffer.h +++ b/src/CellBuffer.h @@ -108,6 +108,7 @@ public: class CellBuffer { private: bool hasStyles; + bool largeDocument; SplitVector substance; SplitVector style; bool readOnly; @@ -126,7 +127,7 @@ private: public: - CellBuffer(bool hasStyles_); + CellBuffer(bool hasStyles_, bool largeDocument_); // Deleted so CellBuffer objects can not be copied. CellBuffer(const CellBuffer &) = delete; void operator=(const CellBuffer &) = delete; @@ -163,6 +164,8 @@ public: bool IsReadOnly() const; void SetReadOnly(bool set); + bool IsLarge() const; + bool HasStyles() const; /// The save point is a marker in the undo stack where the container has stated that /// the buffer was saved. Undo and redo can move over the save point. diff --git a/src/ContractionState.cxx b/src/ContractionState.cxx index 2950587cb..b513c20a6 100644 --- a/src/ContractionState.cxx +++ b/src/ContractionState.cxx @@ -407,8 +407,11 @@ void ContractionState::Check() const { namespace Scintilla { -std::unique_ptr ContractionStateCreate() { - return std::make_unique>(); +std::unique_ptr ContractionStateCreate(bool largeDocument) { + if (largeDocument) + return std::make_unique>(); + else + return std::make_unique>(); } } diff --git a/src/ContractionState.h b/src/ContractionState.h index a951e0a0c..90f5c0784 100644 --- a/src/ContractionState.h +++ b/src/ContractionState.h @@ -45,7 +45,7 @@ public: virtual void ShowAll()=0; }; -std::unique_ptr ContractionStateCreate(); +std::unique_ptr ContractionStateCreate(bool largeDocument); } diff --git a/src/Document.cxx b/src/Document.cxx index 09a91d469..4765248d1 100644 --- a/src/Document.cxx +++ b/src/Document.cxx @@ -82,7 +82,7 @@ int LexInterface::LineEndTypesSupported() { } Document::Document(int options) : - cb((options & SC_DOCUMENTOPTION_STYLES_NONE) == 0) { + cb((options & SC_DOCUMENTOPTION_STYLES_NONE) == 0, (options & SC_DOCUMENTOPTION_TEXT_LARGE) != 0) { refCount = 0; #ifdef _WIN32 eolMode = SC_EOL_CRLF; @@ -113,7 +113,7 @@ Document::Document(int options) : perLineData[ldMargin] = std::make_unique(); perLineData[ldAnnotation] = std::make_unique(); - decorations = DecorationListCreate(false); + decorations = DecorationListCreate(IsLarge()); cb.SetPerLine(this); } @@ -1520,6 +1520,11 @@ void Document::ConvertLineEnds(int eolModeSet) { } +int Document::Options() const { + return (IsLarge() ? SC_DOCUMENTOPTION_TEXT_LARGE : 0) | + (cb.HasStyles() ? 0 : SC_DOCUMENTOPTION_STYLES_NONE); +} + bool Document::IsWhiteLine(Sci::Line line) const { Sci::Position currentChar = static_cast(LineStart(line)); const Sci::Position endLine = static_cast(LineEnd(line)); diff --git a/src/Document.h b/src/Document.h index 7614373e2..d5d0a4d2e 100644 --- a/src/Document.h +++ b/src/Document.h @@ -352,6 +352,8 @@ public: void ConvertLineEnds(int eolModeSet); void SetReadOnly(bool set) { cb.SetReadOnly(set); } bool IsReadOnly() const { return cb.IsReadOnly(); } + bool IsLarge() const { return cb.IsLarge(); } + int Options() const; void DelChar(Sci::Position pos); void DelCharBack(Sci::Position pos); diff --git a/src/EditModel.cxx b/src/EditModel.cxx index 7782f583e..e7400ca37 100644 --- a/src/EditModel.cxx +++ b/src/EditModel.cxx @@ -70,7 +70,7 @@ EditModel::EditModel() { wrapWidth = LineLayout::wrapWidthInfinite; pdoc = new Document(SC_DOCUMENTOPTION_DEFAULT); pdoc->AddRef(); - pcs = ContractionStateCreate(); + pcs = ContractionStateCreate(pdoc->IsLarge()); } EditModel::~EditModel() { diff --git a/src/EditView.cxx b/src/EditView.cxx index 37a6a9f9e..3f490af5e 100644 --- a/src/EditView.cxx +++ b/src/EditView.cxx @@ -2330,16 +2330,15 @@ Sci::Position EditView::FormatRange(bool draw, Sci_RangeToFormat *pfr, Surface * if (draw && lineNumberWidth && (ypos + vsPrint.lineHeight <= pfr->rc.bottom) && (visibleLine >= 0)) { - char number[100]; - sprintf(number, "%d" lineNumberPrintSpace, lineDoc + 1); + const std::string number = std::to_string(lineDoc + 1) + lineNumberPrintSpace; PRectangle rcNumber = rcLine; rcNumber.right = rcNumber.left + lineNumberWidth; // Right justify rcNumber.left = rcNumber.right - surfaceMeasure->WidthText( - vsPrint.styles[STYLE_LINENUMBER].font, number, static_cast(strlen(number))); + vsPrint.styles[STYLE_LINENUMBER].font, number.c_str(), static_cast(number.length())); surface->FlushCachedState(); surface->DrawTextNoClip(rcNumber, vsPrint.styles[STYLE_LINENUMBER].font, - static_cast(ypos + vsPrint.maxAscent), number, static_cast(strlen(number)), + static_cast(ypos + vsPrint.maxAscent), number.c_str(), static_cast(number.length()), vsPrint.styles[STYLE_LINENUMBER].fore, vsPrint.styles[STYLE_LINENUMBER].back); } diff --git a/src/Editor.cxx b/src/Editor.cxx index 0a2cf9c7c..ca23b0030 100644 --- a/src/Editor.cxx +++ b/src/Editor.cxx @@ -5212,7 +5212,7 @@ void Editor::SetDocPointer(Document *document) { pdoc = document; } pdoc->AddRef(); - pcs = ContractionStateCreate(); + pcs = ContractionStateCreate(pdoc->IsLarge()); // Ensure all positions within document sel.Clear(); @@ -7604,7 +7604,7 @@ sptr_t Editor::WndProc(unsigned int iMessage, uptr_t wParam, sptr_t lParam) { Document *doc = new Document(static_cast(lParam)); doc->AddRef(); doc->Allocate(static_cast(wParam)); - pcs = ContractionStateCreate(); + pcs = ContractionStateCreate(pdoc->IsLarge()); return reinterpret_cast(doc); } @@ -7616,12 +7616,15 @@ sptr_t Editor::WndProc(unsigned int iMessage, uptr_t wParam, sptr_t lParam) { (reinterpret_cast(lParam))->Release(); break; + case SCI_GETDOCUMENTOPTIONS: + return pdoc->Options(); + case SCI_CREATELOADER: { Document *doc = new Document(static_cast(lParam)); doc->AddRef(); doc->Allocate(static_cast(wParam)); doc->SetUndoCollection(false); - pcs = ContractionStateCreate(); + pcs = ContractionStateCreate(pdoc->IsLarge()); return reinterpret_cast(static_cast(doc)); } diff --git a/src/MarginView.cxx b/src/MarginView.cxx index a7f9a9a2d..02a961b10 100644 --- a/src/MarginView.cxx +++ b/src/MarginView.cxx @@ -367,10 +367,12 @@ void MarginView::PaintMargin(Surface *surface, Sci::Line topLine, PRectangle rc, rcMarker.bottom = static_cast(yposScreen + vs.lineHeight); if (vs.ms[margin].style == SC_MARGIN_NUMBER) { if (firstSubLine) { - char number[100] = ""; - if (lineDoc >= 0) - sprintf(number, "%d", lineDoc + 1); + std::string sNumber; + if (lineDoc >= 0) { + sNumber = std::to_string(lineDoc + 1); + } if (model.foldFlags & (SC_FOLDFLAG_LEVELNUMBERS | SC_FOLDFLAG_LINESTATE)) { + char number[100] = ""; if (model.foldFlags & SC_FOLDFLAG_LEVELNUMBERS) { const int lev = model.pdoc->GetLevel(lineDoc); sprintf(number, "%c%c %03X %03X", @@ -383,14 +385,15 @@ void MarginView::PaintMargin(Surface *surface, Sci::Line topLine, PRectangle rc, const int state = model.pdoc->GetLineState(lineDoc); sprintf(number, "%0X", state); } + sNumber = number; } PRectangle rcNumber = rcMarker; // Right justify - const XYPOSITION width = surface->WidthText(fontLineNumber, number, static_cast(strlen(number))); + const XYPOSITION width = surface->WidthText(fontLineNumber, sNumber.c_str(), static_cast(sNumber.length())); const XYPOSITION xpos = rcNumber.right - width - vs.marginNumberPadding; rcNumber.left = xpos; DrawTextNoClipPhase(surface, rcNumber, vs.styles[STYLE_LINENUMBER], - rcNumber.top + vs.maxAscent, number, static_cast(strlen(number)), drawAll); + rcNumber.top + vs.maxAscent, sNumber.c_str(), static_cast(sNumber.length()), drawAll); } else if (vs.wrapVisualFlags & SC_WRAPVISUALFLAG_MARGIN) { PRectangle rcWrapMarker = rcMarker; rcWrapMarker.right -= wrapMarkerPaddingRight; diff --git a/src/Position.h b/src/Position.h index a8fbfb494..e0bbcb53f 100644 --- a/src/Position.h +++ b/src/Position.h @@ -16,8 +16,8 @@ namespace Sci { -typedef int Position; -typedef int Line; +typedef ptrdiff_t Position; +typedef ptrdiff_t Line; const Position invalidPosition = -1; diff --git a/src/RunStyles.cxx b/src/RunStyles.cxx index 03692f673..f92da5047 100644 --- a/src/RunStyles.cxx +++ b/src/RunStyles.cxx @@ -306,3 +306,5 @@ void RunStyles::Check() const { template class Scintilla::RunStyles; template class Scintilla::RunStyles; +template class Scintilla::RunStyles; +template class Scintilla::RunStyles; diff --git a/test/unit/testCellBuffer.cxx b/test/unit/testCellBuffer.cxx index f0192423c..cef88cb17 100644 --- a/test/unit/testCellBuffer.cxx +++ b/test/unit/testCellBuffer.cxx @@ -26,7 +26,7 @@ TEST_CASE("CellBuffer") { const char sText[] = "Scintilla"; const Sci::Position sLength = static_cast(strlen(sText)); - CellBuffer cb(true); + CellBuffer cb(true, false); SECTION("InsertOneLine") { bool startSequence = false; diff --git a/test/unit/testContractionState.cxx b/test/unit/testContractionState.cxx index b0346dfc9..fcee90287 100644 --- a/test/unit/testContractionState.cxx +++ b/test/unit/testContractionState.cxx @@ -25,7 +25,7 @@ using namespace Scintilla; TEST_CASE("ContractionState") { - std::unique_ptr pcs = ContractionStateCreate(); + std::unique_ptr pcs = ContractionStateCreate(false); SECTION("IsEmptyInitially") { REQUIRE(1 == pcs->LinesInDoc()); diff --git a/test/unit/testPartitioning.cxx b/test/unit/testPartitioning.cxx index 69bbe7957..449253ef3 100644 --- a/test/unit/testPartitioning.cxx +++ b/test/unit/testPartitioning.cxx @@ -50,7 +50,7 @@ TEST_CASE("SplitVectorWithRangeAdd") { TEST_CASE("Partitioning") { - Partitioning part(growSize); + Partitioning part(growSize); SECTION("IsEmptyInitially") { REQUIRE(1 == part.Partitions()); -- cgit v1.2.3