Fix problem with flex scanners and huge token sizes

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@5410 91177308-0d34-0410-b5e6-96231b3b80d8

diff --git a/Makefile.common b/Makefile.common
index 4d61ad7024d3b81d9b74bab5919b269e6cc1c1ac..37cce15f70fd147f1af95f86719e310115cda08c 100644 (file)
--- a/Makefile.common
+++ b/Makefile.common
@@ -467,8 +467,17 @@ YACC_OUTPUT = $(addprefix $(YACC_FILES:%.y=%), .h .cpp .output)
 
 # Create a .cpp source file from a flex input file... this uses sed to cut down
 # on the warnings emited by GCC...
+#
+# The last line is a gross hack to work around flex aparently not being able to
+# resize the buffer on a large token input.  Currently, for uninitialized string
+# buffers in LLVM we can generate very long tokens, so this is a hack around it.
+# FIXME.  (f.e. char Buffer[10000]; )
+#
 %.cpp: %.l
-       flex -t $< | sed '/^find_rule/d' | sed 's/void yyunput/inline void yyunput/' | sed 's/void \*yy_flex_realloc/inline void *yy_flex_realloc/' > $@
+       flex -t $< | sed '/^find_rule/d' | \
+                     sed 's/void yyunput/inline void yyunput/' | \
+          sed 's/void \*yy_flex_realloc/inline void *yy_flex_realloc/' | \
+          sed 's/#define YY_BUF_SIZE 16384/#define YY_BUF_SIZE (16384*64)/' > $@
 
 # Rule for building the bison parsers...
 

diff --git a/Makefile.rules b/Makefile.rules
index 4d61ad7024d3b81d9b74bab5919b269e6cc1c1ac..37cce15f70fd147f1af95f86719e310115cda08c 100644 (file)
--- a/Makefile.rules
+++ b/Makefile.rules
@@ -467,8 +467,17 @@ YACC_OUTPUT = $(addprefix $(YACC_FILES:%.y=%), .h .cpp .output)
 
 # Create a .cpp source file from a flex input file... this uses sed to cut down
 # on the warnings emited by GCC...
+#
+# The last line is a gross hack to work around flex aparently not being able to
+# resize the buffer on a large token input.  Currently, for uninitialized string
+# buffers in LLVM we can generate very long tokens, so this is a hack around it.
+# FIXME.  (f.e. char Buffer[10000]; )
+#
 %.cpp: %.l
-       flex -t $< | sed '/^find_rule/d' | sed 's/void yyunput/inline void yyunput/' | sed 's/void \*yy_flex_realloc/inline void *yy_flex_realloc/' > $@
+       flex -t $< | sed '/^find_rule/d' | \
+                     sed 's/void yyunput/inline void yyunput/' | \
+          sed 's/void \*yy_flex_realloc/inline void *yy_flex_realloc/' | \
+          sed 's/#define YY_BUF_SIZE 16384/#define YY_BUF_SIZE (16384*64)/' > $@
 
 # Rule for building the bison parsers...