CCM::Tokenizer Class Reference

Inheritance diagram for CCM::Tokenizer:

List of all members.

---

Detailed Description

Tokenizer for C# source code.

Public Member Functions
	Tokenizer (System.IO.TextReader input, string fname, ArrayList defs)
bool	advance ()
	move on to next token.
Object	value ()
	associated with current token.
int	token ()
	classifies current token.
int	xtoken ()
Public Attributes
string	ref_name
int	ref_line = 1
int	line = 1
int	col = 1
int	current_token
Properties
string	location
bool	PropertyParsing
bool	AssemblyTargetParsing
bool	EventParsing
Hashtable	keywords
NumberStyles	styles
NumberFormatInfo	csharp_format_info
int	putback_char
Object	val
Hashtable	defines
const int	TAKING = 1
const int	TAKEN_BEFORE = 2
const int	ELSE_SEEN = 4
const int	PARENT_TAKING = 8
Stack	ifstack
System.Text.StringBuilder	id_builder
System.Text.StringBuilder	string_builder
System.Text.StringBuilder	number_builder
string	error_details
string	error
int	Line
int	Col
Location	Location
Object	Value
StringBuilder	static_cmd_arg = new System.Text.StringBuilder ()
Private Member Functions
bool	is_keyword (string name)
int	GetKeyword (string name)
void	define (string def)
bool	is_identifier_start_character (char c)
bool	is_identifier_part_character (char c)
int	is_punct (char c, ref bool doread)
bool	decimal_digits (int c)
bool	is_hex (char e)
void	hex_digits (int c)
int	real_type_suffix (int c)
int	integer_type_suffix (ulong ul, int c)
int	adjust_int (int c)
int	adjust_real (int t)
int	is_number (int c)
int	getHex (int count, out bool error)
int	escape (int c)
int	getChar ()
int	peekChar ()
void	putback (int c)
void	get_cmd_arg (out string cmd, out string arg)
bool	PreProcessLine (string arg)
void	PreProcessDefinition (bool is_define, string arg)
bool	eval_val (string s)
bool	pp_primary (ref string s)
bool	pp_unary (ref string s)
bool	pp_eq (ref string s)
bool	pp_and (ref string s)
bool	pp_expr (ref string s)
bool	eval (string s)
void	Error_InvalidDirective ()
void	Error_UnexpectedDirective (string extra)
void	Error_TokensSeen ()
bool	handle_preprocessing_directive (bool caller_is_taking)
Static Private Member Functions
void	InitTokens ()
	Tokenizer ()
Private Attributes
TextReader	reader
bool	handle_get_set = true
bool	handle_remove_add = false
bool	handle_assembly = false
bool	tokens_seen = false
bool	any_token_seen = false

---

Constructor & Destructor Documentation

CCM::Tokenizer::Tokenizer (   )  [static, private]

CCM::Tokenizer::Tokenizer (  System.IO.TextReader  input, string  fname, ArrayList  defs )

---

Member Function Documentation

int CCM::Tokenizer::adjust_int (  int  c  )  [private]

int CCM::Tokenizer::adjust_real (  int  t  )  [private]

bool CCM::Tokenizer::advance (   )

move on to next token. Returns: false if positioned beyond tokens. Exceptions: IOException  on input error. Implements CCM::yyParser::yyInput.

bool CCM::Tokenizer::decimal_digits (  int  c  )  [private]

void CCM::Tokenizer::define (  string  def  )  [private]

void CCM::Tokenizer::Error_InvalidDirective (   )  [private]

void CCM::Tokenizer::Error_TokensSeen (   )  [private]

void CCM::Tokenizer::Error_UnexpectedDirective (  string  extra  )  [private]

int CCM::Tokenizer::escape (  int  c  )  [private]

bool CCM::Tokenizer::eval (  string  s  )  [private]

bool CCM::Tokenizer::eval_val (  string  s  )  [private]

void CCM::Tokenizer::get_cmd_arg (  out string  cmd, out string  arg )  [private]

int CCM::Tokenizer::getChar (   )  [private]

int CCM::Tokenizer::getHex (  int  count, out bool  error )  [private]

int CCM::Tokenizer::GetKeyword (  string  name  )  [private]

bool CCM::Tokenizer::handle_preprocessing_directive (  bool  caller_is_taking  )  [private]

void CCM::Tokenizer::hex_digits (  int  c  )  [private]

void CCM::Tokenizer::InitTokens (   )  [static, private]

int CCM::Tokenizer::integer_type_suffix (  ulong  ul, int  c )  [private]

bool CCM::Tokenizer::is_hex (  char  e  )  [private]

bool CCM::Tokenizer::is_identifier_part_character (  char  c  )  [private]

bool CCM::Tokenizer::is_identifier_start_character (  char  c  )  [private]

bool CCM::Tokenizer::is_keyword (  string  name  )  [private]

int CCM::Tokenizer::is_number (  int  c  )  [private]

int CCM::Tokenizer::is_punct (  char  c, ref bool  doread )  [private]

int CCM::Tokenizer::peekChar (   )  [private]

bool CCM::Tokenizer::pp_and (  ref string  s  )  [private]

bool CCM::Tokenizer::pp_eq (  ref string  s  )  [private]

bool CCM::Tokenizer::pp_expr (  ref string  s  )  [private]

bool CCM::Tokenizer::pp_primary (  ref string  s  )  [private]

bool CCM::Tokenizer::pp_unary (  ref string  s  )  [private]

void CCM::Tokenizer::PreProcessDefinition (  bool  is_define, string  arg )  [private]

bool CCM::Tokenizer::PreProcessLine (  string  arg  )  [private]

void CCM::Tokenizer::putback (  int  c  )  [private]

int CCM::Tokenizer::real_type_suffix (  int  c  )  [private]

int CCM::Tokenizer::token (   )

classifies current token. Should not be called if advance() returned false. Returns: current token or single character. Implements CCM::yyParser::yyInput.

Object CCM::Tokenizer::value (   )

associated with current token. Should not be called if advance() returned false. Returns: value for token(). Implements CCM::yyParser::yyInput.

int CCM::Tokenizer::xtoken (   )

---

Member Data Documentation

bool CCM::Tokenizer::any_token_seen = false [private]

int CCM::Tokenizer::col = 1

int CCM::Tokenizer::current_token

bool CCM::Tokenizer::handle_assembly = false [private]

bool CCM::Tokenizer::handle_get_set = true [private]

bool CCM::Tokenizer::handle_remove_add = false [private]

int CCM::Tokenizer::line = 1

TextReader CCM::Tokenizer::reader [private]

int CCM::Tokenizer::ref_line = 1

string CCM::Tokenizer::ref_name

bool CCM::Tokenizer::tokens_seen = false [private]

---

Property Documentation

bool CCM::Tokenizer::AssemblyTargetParsing [get, set]

int CCM::Tokenizer::Col [get]

NumberFormatInfo CCM::Tokenizer::csharp_format_info [static, private]

Hashtable CCM::Tokenizer::defines [private]

const int CCM::Tokenizer::ELSE_SEEN = 4 [private]

string CCM::Tokenizer::error [get]

string CCM::Tokenizer::error_details [private]

bool CCM::Tokenizer::EventParsing [get, set]

System.Text.StringBuilder CCM::Tokenizer::id_builder [static, private]

Stack CCM::Tokenizer::ifstack [private]

Hashtable CCM::Tokenizer::keywords [static, private]

int CCM::Tokenizer::Line [get]

Location CCM::Tokenizer::Location [get]

string CCM::Tokenizer::location [get]

System.Text.StringBuilder CCM::Tokenizer::number_builder [static, private]

const int CCM::Tokenizer::PARENT_TAKING = 8 [private]

bool CCM::Tokenizer::PropertyParsing [get, set]

int CCM::Tokenizer::putback_char [private]

StringBuilder CCM::Tokenizer::static_cmd_arg = new System.Text.StringBuilder () [static, private]

System.Text.StringBuilder CCM::Tokenizer::string_builder [static, private]

NumberStyles CCM::Tokenizer::styles [static, private]

const int CCM::Tokenizer::TAKEN_BEFORE = 2 [private]

const int CCM::Tokenizer::TAKING = 1 [private]

Object CCM::Tokenizer::val [private]

Object CCM::Tokenizer::Value [get]

---

The documentation for this class was generated from the following file:

• cs-tokenizer.cs

---