Changeset adfc63c96dc82134b341c1fc225397eea16d4167

Show
Ignore:
Timestamp:
06/26/2008 03:37:23 PM (2 months ago)
Author:
mitchell <mitchell@frost.(none)>
git-committer:
mitchell <mitchell@frost.(none)> 1214519843 -0400
git-parent:

[048a0f28e87a635a3ba3d08cb607095f63e536a9]

git-author:
mitchell <mitchell@frost.(none)> 1214519843 -0400
Message:

Deleted the code the new Ragel parser obsoletes.

Files:

Legend:

Unmodified
Added
Removed
Modified
Copied
Moved

  • README

    r338ef7c radfc63c  
    4848also require a C compiler to build the native extensions. 
    4949 
    50 Ohcount requires the pcre library (http:///www.pcre.org). 
     50Ohcount requires Ragel (http://research.cs.queensu.ca/~thurston/ragel/) 
     51Unfortunately, Ragel 6.2 is not recent enough so you will need the latest 
     52version in SVN: svn://mambo.cs.queensu.ca/ragel/trunk/. 
    5153 
    5254== Download 
    …… 
    118120* Update Ohcount::Detector to identify files that use the new language. 
    119121* Update Ohcount::DetectorTest to confirm the Detector changes. 
    120 * In Ohcount::Generator, instantiate a new Ohcount::Monoglot or 
    121   Ohcount::Polyglot to parse the language. 
     122* Follow the detailed instructions in PARSER_DOC. 
    122123* In Ohcount::SlocInfo, provide a "nice name" and category (procedural 
    123124  code vs. markup) for the new language. 

  • Rakefile

    r9638a53 radfc63c  
    1717CLEAN.include FileList["#{EXT_DIR}/*.{so,bundle,#{CONFIG['DLEXT']}}"], 
    1818              FileList["#{EXT_DIR}/*.o"], 
    19               FileList["#{EXT_DIR}/polyglots.c"], 
    2019              FileList["#{EXT_DIR}/Makefile"], 
    2120              (FileList["#{EXT_DIR}/*_parser.h"] - FileList["#{EXT_DIR}/ragel_parser.h"]) 
    …… 
    2524PKG_FILES = %w(README COPYING Rakefile lib/ohcount.rb) + 
    2625  Dir.glob("ext/ohcount_native/*.{h,c,rb}") + 
    27   Dir.glob("ext/ohcount_native/glots/*.rb") + 
    2826  Dir.glob("lib/ohcount/*.rb") + 
    2927  Dir.glob("test/*") + 
    …… 
    6462end 
    6563 
    66 file EXT_DL => FileList["#{EXT_DIR}/polyglots.c", "#{EXT_DIR}/Makefile", "#{EXT_DIR}/*.{c,h,rb}"] do 
     64file EXT_DL => FileList["#{EXT_DIR}/Makefile", "#{EXT_DIR}/*.{c,h,rb}"] do 
    6765  cd EXT_DIR do 
    6866    cd 'ragel_parsers' do 
    …… 
    9593end 
    9694 
    97 file "#{EXT_DIR}/polyglots.c" => FileList["#{EXT_DIR}/*.rb", "#{EXT_DIR}/glots/*.rb"] do 
    98   cd EXT_DIR do 
    99     ruby 'generator.rb' 
    100   end 
    101 end 
    102  
    10395Rake::RDocTask.new do |rdoc| 
    10496  rdoc.rdoc_dir = 'doc' 

  • ext/ohcount_native/common.h

    r069dddc radfc63c  
    1313 Limits 
    1414*******************************************/ 
    15 // The Parser's CompiledState Stack 
    16 #define MAX_CS_STACK 20 
    1715// Parser's Maximum number of LanguageBreakdowns it can return 
    1816#define MAX_LANGUAGE_BREAKDOWN_SIZE 8 
    19 // How large can a CompiledState's regex term be? 
    20 #define MAX_REGEX 200 
    21 // CompiledState's number of transitions 
    22 #define MAX_TRANSITIONS 10 
    2317// The longest a language name can be 
    2418#define MAX_LANGUAGE_NAME 20 
    …… 
    3024#include <stdio.h> 
    3125#include <string.h> 
    32 #include <pcre.h> 
    33 #include "transition.h" 
    34 #include "state.h" 
    35 #include "compiled_state.h" 
    36 #include "polyglot.h" 
    37 #include "polyglots.h" 
    3826#include "language_breakdown.h" 
    39 #include "parser.h" 
    4027#include "ragel_parser.h" 
    41  
    42 /******************************************* 
    43  Error Handling 
    44 *******************************************/ 
    45 void die(char *err, int exit_code); 
    46  
    47 enum EXIT_CODES { 
    48   ERR_PCRE_OUT_OF_MEMORY = 15, 
    49   ERR_PCRE_GENERIC, 
    50   ERR_UNKNOWN_SEMANTIC 
    51 }; 
    52  
    5328 
    5429/******************************************* 

  • ext/ohcount_native/extconf.rb

    rb4f9575 radfc63c  
    33 
    44dir_config('ohcount_native') 
    5 have_library('pcre','pcre_compile') 
    65 
    76# FLAGS: enable logging (or not) 

  • ext/ohcount_native/ragel_parser.h

    r069dddc radfc63c  
     1// ragel_parser.h written by Mitchell Foral. mitchell<att>caladbolg<dott>net. 
     2 
     3/** 
     4 * Each language (html, css, etc.) is represented in its own language_breakdown. 
     5 */ 
     6typedef struct { 
     7  LanguageBreakdown language_breakdowns[MAX_LANGUAGE_BREAKDOWN_SIZE]; 
     8  int language_breakdown_count; 
     9} ParseResult; 
     10 
     11 
     12/** 
     13 * Fills out the ParseResult with the result of parsing the buffer with the specific Language. 
     14 */ 
    115int ragel_parser_parse(ParseResult *pr, int count, char *buf, int buf_len, char *lang); 

  • ext/ohcount_native/ragel_parser_macros.h

    r5b7fcf9 radfc63c  
     1// ragel_parser_macros.h written by Mitchell Foral. mitchell<att>caladbolg<dott>net 
     2 
    13#ifndef RAGEL_PARSER_MACROS 
    24#define RAGEL_PARSER_MACROS 

  • ext/ohcount_native/ruby_binding.c

    r069dddc radfc63c  
    122122 */ 
    123123static VALUE _ohcount_parse(VALUE self, VALUE buffer, VALUE polyglot_name_value) { 
    124  
    125   // find the polyglot to parse with 
     124  ParseResult pr; 
     125 
    126126  char *polyglot_name = RSTRING(polyglot_name_value)->ptr; 
    127   int i_polyglot; 
    128   for (i_polyglot = 0; POLYGLOTS[i_polyglot] != NULL; i_polyglot++) { 
    129     if (strcmp(POLYGLOTS[i_polyglot]->name, polyglot_name) == 0) { 
    130       Polyglot *polyglot = POLYGLOTS[i_polyglot]; 
    131  
    132       ParseResult pr; 
    133       parser_parse(&pr, RSTRING(buffer)->ptr, RSTRING(buffer)->len, polyglot); 
    134  
    135       // create array we'll return all the language_breakdowns in 
    136       VALUE ary = rb_ary_new2(pr.language_breakdown_count); 
    137  
    138       int i_pr; 
    139       for(i_pr = 0; i_pr < pr.language_breakdown_count; i_pr++) { 
    140         LanguageBreakdown *lb = (LanguageBreakdown *) malloc(sizeof(LanguageBreakdown)); 
    141         LanguageBreakdown *src_lb = &(pr.language_breakdowns[i_pr]); 
    142         strcpy(lb->name,src_lb->name); 
    143         lb->code = src_lb->code; 
    144         lb->comment = src_lb->comment; 
    145         lb->blank_count = src_lb->blank_count; 
    146         rb_ary_store(ary, i_pr, Data_Wrap_Struct(rb_class_language_breakdown, 0, _language_breakdown_free, lb)); 
    147       } 
    148  
    149       return ary; 
     127  if (ragel_parser_parse(&pr, 1, RSTRING(buffer)->ptr, RSTRING(buffer)->len, polyglot_name)) { 
     128    // create array we'll return all the language_breakdowns in 
     129    VALUE ary = rb_ary_new2(pr.language_breakdown_count); 
     130 
     131    int i_pr; 
     132    for(i_pr = 0; i_pr < pr.language_breakdown_count; i_pr++) { 
     133      LanguageBreakdown *lb = (LanguageBreakdown *) malloc(sizeof(LanguageBreakdown)); 
     134      LanguageBreakdown *src_lb = &(pr.language_breakdowns[i_pr]); 
     135      strcpy(lb->name,src_lb->name); 
     136      lb->code = src_lb->code; 
     137      lb->comment = src_lb->comment; 
     138      lb->blank_count = src_lb->blank_count; 
     139      rb_ary_store(ary, i_pr, Data_Wrap_Struct(rb_class_language_breakdown, 0, _language_breakdown_free, lb)); 
    150140    } 
     141 
     142    return ary; 
    151143  } 
    152144  rb_raise(rb_eStandardError,"Polyglot name invalid: '%s'", polyglot_name); 
    153145  return Qnil; 
    154 } 
    155  
    156  
    157 static VALUE _ohcount_polyglots(VALUE self) { 
    158  
    159   // how many are they? 
    160   int poly_count = 0; 
    161   Polyglot **p = POLYGLOTS; 
    162   while ((*p++) != NULL) { 
    163     poly_count++; 
    164   } 
    165  
    166   // create the array 
    167   VALUE ary = rb_ary_new2(poly_count); 
    168  
    169   // fill it in 
    170   int i_poly; 
    171   for (i_poly = 0; POLYGLOTS[i_poly] != NULL; i_poly++) { 
    172     VALUE poly_name = rb_str_new2(POLYGLOTS[i_poly]->name); 
    173     rb_ary_store(ary, i_poly, poly_name); 
    174   } 
    175  
    176   return ary; 
    177146} 
    178147 
    …… 
    219188  rb_define_module_function(rb_module_ohcount, "parse", _ohcount_parse, 2); 
    220189  rb_define_module_function(rb_module_ohcount, "parse_entities", _ohcount_parse_entities, 2); 
    221   rb_define_module_function(rb_module_ohcount, "polyglots", _ohcount_polyglots, 0); 
    222190 
    223191  // define language_breakdown