# HG changeset patch
# User Michael Stahl <mst@openoffice.org>
# Date 1303140561 0
# Node ID 5fdd9cee833c7ef9b4651c5dde05fda22abbc931
# Parent  1cbf1719c76b9f60899c04686ea61d24d5af7f35
rtftok01: writerfilter: refactor flex lexer to use flex "reentrant" option:
 RTFScanner.hxx: replace class RTFScanner with a function scan().
 RTFScanner.lex: refactor completely to replace the various customizations
    with ordinary flex option "reentrant", and also fix several bugs.

diff --git a/writerfilter/inc/rtftok/RTFScanner.hxx b/writerfilter/inc/rtftok/RTFScanner.hxx
--- a/writerfilter/inc/rtftok/RTFScanner.hxx
+++ b/writerfilter/inc/rtftok/RTFScanner.hxx
@@ -29,37 +29,15 @@
 #define INCLUDED_RTFSCANNER_HXX
 
 #include <WriterFilterDllApi.hxx>
-#include <vector>
-#include <rtftok/RTFInputSource.hxx>
-#include <rtftok/RTFScannerHandler.hxx>
 
 namespace writerfilter { namespace rtftok {
 
-class WRITERFILTER_DLLPUBLIC RTFScanner {
-public:
-        RTFScanner(RTFScannerHandler &eventHandler_) : eventHandler(eventHandler_) {};
-        virtual ~RTFScanner()	{ }
+class RTFInputSource;
+class RTFScannerHandler;
 
-	const char* YYText()	{ return yytext; }
-	int YYLeng()		{ return yyleng; }
-
-	virtual int yylex() = 0;
-
-	int lineno() const		{ return yylineno; }
-
-protected:
-	char* yytext;
-	int yyleng;
-	int yylineno;		// only maintained if you use %option yylineno
- 	int yy_flex_debug;	// only has effect with -d or "%option debug"
-        RTFScannerHandler &eventHandler;
-
-
-public:
-  static writerfilter::rtftok::RTFScanner* createRTFScanner(writerfilter::rtftok::RTFInputSource& inputSource, writerfilter::rtftok::RTFScannerHandler &eventHandler);
-};
+int WRITERFILTER_DLLPUBLIC
+scan(RTFInputSource & rInputSource, RTFScannerHandler & rEventHandler);
 
 } } /* end namespace writerfilter::rtftok */
 
-
 #endif /* INCLUDED_RTFSCANNER_HXX */
diff --git a/writerfilter/source/rtftok/RTFScanner.lex b/writerfilter/source/rtftok/RTFScanner.lex
--- a/writerfilter/source/rtftok/RTFScanner.lex
+++ b/writerfilter/source/rtftok/RTFScanner.lex
@@ -1,7 +1,7 @@
 /*************************************************************************
  *
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- * 
+ *
  * Copyright 2000, 2010 Oracle and/or its affiliates.
  *
  * OpenOffice.org - a multi-platform office productivity suite
@@ -25,86 +25,122 @@
  *
  ************************************************************************/
 
-/* compile with flex++ -8 -f -+ -Sflex.skl -ortfparser.cxx rtfparser.lex */
+/* OPTIONS */
+/* we do not read several files with one lexer, so no need for yywrap */
+%option noyywrap
+%option 8bit
+%option reentrant
 %option yylineno
 %{
-  //#include <io.h>
-#include <math.h>
-#include <string.h>
-#include <osl/file.h>
-#include <assert.h>
-#include <vector>
+
+#include <strings.h>
+
+#include <algorithm>
 
 #if defined (UNX)
   #define stricmp strcasecmp
 #endif
 
-writerfilter::rtftok::RTFScanner* writerfilter::rtftok::RTFScanner::createRTFScanner(class writerfilter::rtftok::RTFInputSource& inputSource, writerfilter::rtftok::RTFScannerHandler &eventHandler)
+#include <rtftok/RTFInputSource.hxx>
+#include <rtftok/RTFScannerHandler.hxx>
+
+#include <com/sun/star/uno/Any.hxx>
+
+#include <com/sun/star/lang/WrappedTargetException.hpp>
+
+
+struct RTFFlexLexerState
 {
-  return new yyFlexLexer(&inputSource, eventHandler);
+public:
+    RTFFlexLexerState(writerfilter::rtftok::RTFScannerHandler & rHandler,
+            writerfilter::rtftok::RTFInputSource & rInputSource)
+        : m_rHandler(rHandler)
+        , m_rInputSource(rInputSource)
+        , m_nLines()
+        , m_nChars()
+        , m_bStarFlag()
+    { }
+    writerfilter::rtftok::RTFScannerHandler & m_rHandler;
+    writerfilter::rtftok::RTFInputSource & m_rInputSource;
+    ::com::sun::star::uno::Any m_Error;    /// caught execption
+    int m_nLines;
+    sal_uInt64 m_nChars;
+    bool m_bStarFlag;
+};
+
+#define YY_EXTRA_TYPE RTFFlexLexerState *
+
+
+#define YY_INPUT(buf, result, max_size) \
+{ \
+    try { \
+        result = yyextra->m_rInputSource.read(buf, max_size); \
+    } catch (::com::sun::star::uno::Exception & rException) { \
+        yyextra->m_Error <<= rException; \
+        result = 0; \
+    } catch (...) { \
+        OSL_TRACE("caught something..."); \
+        result = 0; \
+    } \
 }
 
 
-
-extern "C" {
-//int isatty(int fd) { return 0; }
-int yywrap(void) { return 1; }
+static void
+make_string(char const*const yytext_, size_t const start, size_t const end,
+           char *& o_rpTarget)
+{
+    o_rpTarget = new char[end - start + 1];
+    ::std::copy(&yytext_[start], &yytext_[end], o_rpTarget);
+    o_rpTarget[end - start] = '\0';
 }
 
-/*
-oslFileHandle yy_osl_in=NULL;
-#define YY_INPUT(buf,result,max_size) \
-{\
-{\
-	assert(yy_osl_in!=NULL);\
-	sal_Bool isEOF;\
-	oslFileError ret=osl_isEndOfFile( yy_osl_in, &isEOF );\
-	assert(ret==osl_File_E_None);\
-	if (isEOF)\
-	{\
-		result=YY_NULL;\
-	}\
-	else\
-	{\
-	sal_uInt64 bytesRead;\
-	ret=osl_readFile( yy_osl_in, buf, max_size, &bytesRead);\
-	assert(ret==osl_File_E_None);\
-	result = bytesRead; \
-	}\
-}\
+static void
+split_token_value(
+    char const*const yytext_, char *& o_rpToken, char *& o_rpValue)
+{
+    size_t i = 0; // skip first '\'
+    while (yytext_[i] != 0 && (yytext_[i] == '\r' || yytext_[i] == '\n')) {i++;}
+    while (yytext_[i] != 0 &&
+         (yytext_[i] < 'A' || (yytext_[i] > 'Z' && yytext_[i] < 'a') ||
+          yytext_[i] > 'z'))
+    { i++; }
+    size_t start = i;
+    while (yytext_[i] != 0 && yytext_[i] >= 'A') { i++; }
+    make_string(yytext_, start, i, o_rpToken);
+    start = i;
+    while (yytext_[i] !=0 && yytext_[i] > ' ') { i++; }
+    make_string(yytext_, start, i, o_rpValue);
 }
-*/
 
-//extern RtfTokenizer* this;
-void yyFlexLexer::split_ctrl(char *_yytext, char* token, char *value)
-   {
-     int i=0; // skip first '\'
-     while(_yytext[i]!=0 && (_yytext[i]=='\r' || _yytext[i]=='\n')) i++;
-     while(_yytext[i]!=0 && (_yytext[i]<'A' || (_yytext[i]>'Z' && _yytext[i]<'a') || _yytext[i]>'z')) i++; 
-     while(_yytext[i]!=0 && _yytext[i]>='A') *(token++)=_yytext[i++];
-     *token=0;
-     while(_yytext[i]!=0 && _yytext[i]>' ') *(value++)=_yytext[i++];
-     *value=0;
-   }
+static void
+raise_ctrl(writerfilter::rtftok::RTFScannerHandler & rHandler, char * yytext_)
+{
+    char * pToken(0);
+    char * pValue(0);
+    split_token_value(yytext_, pToken, pValue);
+    if (pToken && pValue) {
+        rHandler.ctrl(pToken, pValue);
+    }
+    delete[] pToken;
+    delete[] pValue;
+}
 
- void yyFlexLexer::raise_ctrl(char* _yytext)
-   {
-     char token[50];
-     char value[50];
-     split_ctrl(_yytext, token, value);
-     eventHandler.ctrl(token, value);
-   }
+static void
+raise_dest(writerfilter::rtftok::RTFScannerHandler & rHandler, char * yytext_)
+{
+    char * pToken(0);
+    char * pValue(0);
+    split_token_value(yytext_, pToken, pValue);
+    if (pToken && pValue) {
+        rHandler.dest(pToken, pValue);
+    }
+    delete[] pToken;
+    delete[] pValue;
+}
 
- void yyFlexLexer::raise_dest(char* _yytext)
-   {
-     char token[50];
-     char value[50];
-     split_ctrl(_yytext, token, value);
-     eventHandler.dest(token, value);
-   }
- 
-#define _num_of_destctrls (sizeof(_destctrls)/sizeof(_destctrls[0]))
-static const char* _destctrls[] = {
+
+#define NUM_OF_DESTCTRLS (sizeof(g_destctrls)/sizeof(g_destctrls[0]))
+static char const*const g_destctrls[] = {
 "aftncn",
 "aftnsep",
 "aftnsepc",
@@ -241,93 +277,170 @@
 "upr",
 "urtf",
 "userprops",
-"xe"
+"xe",
 };
 
- void yyFlexLexer::raise_destOrCtrl(char* _yytext)
-   {
-     char token[50];
-     char value[50];
-     split_ctrl(_yytext, token, value);
-     char* result=(char*)bsearch(token, _destctrls, _num_of_destctrls, 20, (int (*)(const void*, const void*))stricmp);
-     if (result)
-       {
-	 eventHandler.dest(token, value);
-       }
-     else 
-       {
-	 eventHandler.lbrace();
-	 eventHandler.ctrl(token, value);
-       }
-   }
+struct case_compare
+{
+    bool operator()(char const* const left, char const* const right)
+    {
+        return (strcasecmp(left, right) < 0) ? true : false;
+    }
+};
+
+static void
+raise_destOrCtrl(
+        writerfilter::rtftok::RTFScannerHandler & rHandler, char * yytext_)
+{
+    char * pToken(0);
+    char * pValue(0);
+    split_token_value(yytext_, pToken, pValue);
+    if (pToken && pValue) {
+        bool const bFound( ::std::binary_search(
+            &g_destctrls[0], &g_destctrls[NUM_OF_DESTCTRLS], pToken,
+            case_compare()) );
+        if (bFound) {
+            rHandler.dest(pToken, pValue);
+        } else {
+            rHandler.lbrace();
+            rHandler.ctrl(pToken, pValue);
+        }
+    }
+    delete[] pToken;
+    delete[] pValue;
+}
 
 %}
 
 %%
+
 \{\\upr\{" "? { /* skip upr destination */
-  int c;
-  int br=1;
-  while (br>0 && (c = yyinput()) != EOF)
-    {
-      if (c=='}') br--;
-      if (c=='{') br++;
+    int br = 1;
+    int c;
+    do {
+        c = yyinput(yyscanner);
+        if      (c == '}') { br--; }
+        else if (c == '{') { br++; }
     }
-  eventHandler.lbrace();
-  num_chars+=yyleng;
+    while ((c != EOF) && (br > 0));
+    yyextra->m_rHandler.lbrace();
+    yyextra->m_nChars += yyleng;
 }
 
 
 \\bin(("+"|"-")?[0-9]*)?" "? {
-  raise_dest(yytext);
-  num_chars+=yyleng;
-  int len=atoi(yytext+4);
-   num_chars+=len;
-   //   pictureBytes=2*len;
-  while ( len )
+    raise_dest(yyextra->m_rHandler, yytext);
+    yyextra->m_nChars += yyleng;
+    int len = atoi(yytext+4);
+    yyextra->m_nChars += len;
+    while (len)
     {
-      int c = yyinput();
-      eventHandler.addBinData((unsigned char)c);
-      len--;      
+        int c = yyinput(yyscanner);
+        if (EOF == c) {
+            OSL_TRACE("RTF lexer: binary data shorter than expected");
+            ::com::sun::star::uno::Exception ex;
+            yyextra->m_Error <<= ex;
+            yyterminate();
+        }
+        yyextra->m_rHandler.addBinData(static_cast<unsigned char>(c));
+        len--;
     }
-  eventHandler.rbrace();
+    yyextra->m_rHandler.rbrace();
 }
 
 \{[\r\n]*\\\*\\[A-Za-z]+(("+"|"-")?[0-9]*)?" "? { /* stared dest word */
-        raise_dest(yytext);
-        num_chars+=yyleng;
+    raise_dest(yyextra->m_rHandler, yytext);
+    yyextra->m_nChars += yyleng;
 }
 \{[\r\n]*\\[A-Za-z]+(("+"|"-")?[0-9]*)?" "? { /* dest word */
-        raise_destOrCtrl(yytext);
+    raise_destOrCtrl(yyextra->m_rHandler, yytext);
+    yyextra->m_nChars += yyleng;
 }
 \\[A-Za-z]+(("+"|"-")?[0-9]*)?" "? { /* ctrl word */
-        raise_ctrl(yytext);
-	star_flag=0;
-	num_chars+=yyleng;
+    raise_ctrl(yyextra->m_rHandler, yytext);
+    yyextra->m_bStarFlag = 0;
+    yyextra->m_nChars += yyleng;
 }
 \\\'[A-Fa-f0-9][A-Fa-f0-9] { /* hex char */
-  eventHandler.addHexChar(yytext);
-  num_chars+=yyleng;
+    yyextra->m_rHandler.addHexChar(yytext);
+    yyextra->m_nChars += yyleng;
 }
-\\* { /* star */
-	star_flag=1;
-	num_chars+=yyleng;
+\\\* { /* star */
+    yyextra->m_bStarFlag = 1;
+    yyextra->m_nChars += yyleng;
 }
 \{ { /* lbrace */
-        eventHandler.lbrace();
-        num_chars+=yyleng;
+    yyextra->m_rHandler.lbrace();
+    yyextra->m_nChars += yyleng;
 }
 \} { /* rbrace */
-        eventHandler.rbrace();
-        num_chars+=yyleng;
+    yyextra->m_rHandler.rbrace();
+    yyextra->m_nChars += yyleng;
 }
-\\\| { num_chars+=yyleng;}
-\\~ {num_chars+=yyleng; eventHandler.addCharU(0xa0);}
-\\- {num_chars+=yyleng;}
-\\_ {num_chars+=yyleng;}
-\\\: {num_chars+=yyleng;}
-\n   {   ++num_lines;num_chars+=yyleng;}
-\r {num_chars+=yyleng;}
-\t {num_chars+=yyleng;}
-" "(" "+) { eventHandler.addSpaces(yyleng); num_chars+=yyleng;}
-. { eventHandler.addChar(yytext[0]); num_chars+=yyleng;}
+\\\| { /* formula character; probably obsolete */
+    yyextra->m_nChars += yyleng;
+}
+\\~ { /* non breaking space */
+    yyextra->m_rHandler.addCharU(0xa0);
+    yyextra->m_nChars += yyleng;
+}
+\\- { /* optional hyphen */
+    yyextra->m_nChars += yyleng;
+}
+\\_ { /* non breaking hyphen */
+    yyextra->m_nChars += yyleng;
+}
+\\\: { /* subentry (no idea what this does) */
+    yyextra->m_nChars += yyleng;
+}
+\\\\ { /* escaped backslash */
+    yyextra->m_rHandler.addCharU(sal_Unicode('\\'));
+    yyextra->m_nChars += yyleng;
+}
+\\\{ { /* escaped brace */
+    yyextra->m_rHandler.addCharU(sal_Unicode('{'));
+    yyextra->m_nChars += yyleng;
+}
+\\\} { /* escaped brace */
+    yyextra->m_rHandler.addCharU(sal_Unicode('}'));
+    yyextra->m_nChars += yyleng;
+}
+\n   {
+    ++yyextra->m_nLines;
+    yyextra->m_nChars += yyleng;
+}
+\r { yyextra->m_nChars += yyleng; }
+\t { yyextra->m_nChars += yyleng; }
+" "(" "+) {
+    yyextra->m_rHandler.addSpaces(yyleng);
+    yyextra->m_nChars += yyleng;
+}
+. {
+    yyextra->m_rHandler.addChar(yytext[0]);
+    yyextra->m_nChars += yyleng;
+}
+
 %%
+
+namespace writerfilter { namespace rtftok {
+
+int scan(
+    writerfilter::rtftok::RTFInputSource & rInputSource,
+    writerfilter::rtftok::RTFScannerHandler & rEventHandler)
+{
+    RTFFlexLexerState state(rEventHandler, rInputSource);
+    yyscan_t scanner;
+    yylex_init(& scanner);
+    yyset_extra(& state, scanner);
+    int ret = yylex(scanner);
+    yylex_destroy(scanner);
+    if (state.m_Error.hasValue()) {
+        throw ::com::sun::star::lang::WrappedTargetException(
+            ::rtl::OUString(RTL_CONSTASCII_USTRINGPARAM(
+                "exception caught by RTFScanner")), 0, state.m_Error);
+    }
+    return ret;
+}
+
+} } /* end namespace writerfilter::rtftok */
+
diff --git a/writerfilter/source/rtftok/makefile.mk b/writerfilter/source/rtftok/makefile.mk
--- a/writerfilter/source/rtftok/makefile.mk
+++ b/writerfilter/source/rtftok/makefile.mk
@@ -1,7 +1,7 @@
 #************************************************************************
 #
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
-# 
+#
 # Copyright 2000, 2010 Oracle and/or its affiliates.
 #
 # OpenOffice.org - a multi-platform office productivity suite
@@ -55,7 +55,7 @@
 SHL1IMPLIB=i$(SHL1TARGET)
 SHL1USE_EXPORTS=name
 
-SHL1OBJS=$(SLOFILES) 
+SHL1OBJS=$(SLOFILES)
 
 SHL1DEF=$(MISC)$/$(SHL1TARGET).def
 DEF1NAME=$(SHL1TARGET)
@@ -69,8 +69,8 @@
 
 GENERATEDFILES=$(RTFSCANNERCXX)
 
-$(RTFSCANNERCXX): RTFScanner.lex RTFScanner.skl FlexLexer.h
-	flex -+ -SRTFScanner.skl -o$@ RTFScanner.lex
+$(RTFSCANNERCXX): RTFScanner.lex
+	/usr/bin/flex -o$@ RTFScanner.lex
 
 $(SLO)/RTFScanner.obj: $(RTFSCANNERCXX)
 
diff --git a/writerfilter/unocomponent/debugservices/rtftok/ScannerTestService.cxx b/writerfilter/unocomponent/debugservices/rtftok/ScannerTestService.cxx
--- a/writerfilter/unocomponent/debugservices/rtftok/ScannerTestService.cxx
+++ b/writerfilter/unocomponent/debugservices/rtftok/ScannerTestService.cxx
@@ -31,6 +31,7 @@
 #include <wchar.h>
 #include <rtftok/RTFScanner.hxx>
 #include <rtftok/RTFScannerHandler.hxx>
+#include <rtftok/RTFInputSource.hxx>
 #include <com/sun/star/io/XStream.hpp>
 #include <com/sun/star/io/XInputStream.hpp>
 #include <com/sun/star/io/XSeekable.hpp>
@@ -408,10 +409,7 @@
 
 			RtfInputSourceImpl rtfInputSource(xInputStream, xStatusIndicator);
 			MyRtfScannerHandler eventHandler(xServiceFactory, xFileAccess, xStorage);
-			writerfilter::rtftok::RTFScanner *rtfScanner=writerfilter::rtftok::RTFScanner::createRTFScanner(rtfInputSource, eventHandler);
-
-			rtfScanner->yylex();
-			delete rtfScanner;
+            writerfilter::rtftok::scan(rtfInputSource, eventHandler);
 
 		TimeValue t2; osl_getSystemTime(&t2);
 		printf("time=%" SAL_PRIuUINT32 "s\n", t2.Seconds-t1.Seconds);
diff --git a/writerfilter/unocomponent/debugservices/rtftok/XMLScanner.cxx b/writerfilter/unocomponent/debugservices/rtftok/XMLScanner.cxx
--- a/writerfilter/unocomponent/debugservices/rtftok/XMLScanner.cxx
+++ b/writerfilter/unocomponent/debugservices/rtftok/XMLScanner.cxx
@@ -31,6 +31,7 @@
 #include <wchar.h>
 #include <rtftok/RTFScanner.hxx>
 #include <rtftok/RTFScannerHandler.hxx>
+#include <rtftok/RTFInputSource.hxx>
 #include <com/sun/star/io/XStream.hpp>
 #include <com/sun/star/io/XInputStream.hpp>
 #include <com/sun/star/io/XSeekable.hpp>
@@ -292,12 +293,10 @@
 
 			RtfInputSourceImpl rtfInputSource(xInputStream, xStatusIndicator);
 			XmlRtfScannerHandler eventHandler(xServiceFactory, xFileAccess);
-			writerfilter::rtftok::RTFScanner *rtfScanner=writerfilter::rtftok::RTFScanner::createRTFScanner(rtfInputSource, eventHandler);
 
             cout << "<out>" << endl;
-			rtfScanner->yylex();
+            writerfilter::rtftok::scan(rtfInputSource, eventHandler);
             cout << "</out>" << endl;
-			delete rtfScanner;
 
         ::ucbhelper::ContentBroker::deinitialize();
 	}
