tokpar.c File Reference

#include <stdbool.h>
#include <ctype.h>
#include <string.h>
#include <stdlib.h>
#include "lbm_memory.h"
#include "lbm_types.h"
#include "lbm_channel.h"
#include "tokpar.h"
#include "symrepr.h"
#include "heap.h"
#include "env.h"

Include dependency graph for tokpar.c:

Data Structures
struct	matcher

Macros
#define	NUM_FIXED_SIZE_TOKENS   18
#define	NUM_TYPE_QUALIFIERS   9
#define	TD_BUF_SIZE   128
#define	FBUF_ADD(X, N)   if ((N) < TD_BUF_SIZE) { fbuf[(N)] = (X); N++; } else goto tok_double_no_tok;

Functions
static signed char	translate_escape_char (char c)
static int	tok_match_fixed_size_tokens (lbm_char_channel_t *chan, const matcher *m, unsigned int start_pos, unsigned int num, uint32_t *res)
int	tok_syntax (lbm_char_channel_t *chan, uint32_t *res)
static bool	alpha_char (char c)
static bool	num_char (char c)
static bool	symchar0 (char c)
static bool	symchar (char c)
int	tok_symbol (lbm_char_channel_t *chan)
int	tok_string (lbm_char_channel_t *chan, unsigned int *string_len)
int	tok_char (lbm_char_channel_t *chan, char *res)
int	tok_double (lbm_char_channel_t *chan, token_float *result)
bool	tok_clean_whitespace (lbm_char_channel_t *chan)
int	tok_integer (lbm_char_channel_t *chan, token_int *result)

Variables
char	tokpar_sym_str [TOKENIZER_MAX_SYMBOL_AND_STRING_LENGTH+1]
const matcher	fixed_size_tokens [NUM_FIXED_SIZE_TOKENS]
const matcher	type_qual_table [NUM_TYPE_QUALIFIERS]

Macro Definition Documentation

FBUF_ADD

#define FBUF_ADD	(		X,
			N
	)		if ((N) < TD_BUF_SIZE) { fbuf[(N)] = (X); N++; } else goto tok_double_no_tok;

NUM_FIXED_SIZE_TOKENS

#define NUM_FIXED_SIZE_TOKENS   18

NUM_TYPE_QUALIFIERS

#define NUM_TYPE_QUALIFIERS   9

TD_BUF_SIZE

#define TD_BUF_SIZE   128

Function Documentation

alpha_char()

static bool alpha_char ( char  c )

static

num_char()

static bool num_char ( char  c )

static

symchar()

static bool symchar ( char  c )

static

symchar0()

static bool symchar0 ( char  c )

static

tok_char()

int tok_char	(	lbm_char_channel_t *	chan,
		char *	res
	)

Attempt to read a character from the character stream.

Parameters

chan	Character channel to read characters from.
res	Result character if successful.

Returns

A positive value indicating number of characters used if successful. Otherwise a negative status indicator.

tok_clean_whitespace()

bool tok_clean_whitespace

(

lbm_char_channel_t *

chan

)

Clean off whitespace from head of the character stream

Returns

True if whitespace could be cleaned. False if stream has no more available characters at the moment.

tok_double()

int tok_double	(	lbm_char_channel_t *	chan,
		token_float *	result
	)

Attempt to read a floating point number from the character stream.

Parameters

chan	Character channel to read characters from.
res	Result token_float-object if successful.

Returns

A positive value indicating number of characters used if successful. Otherwise a negative status indicator.

tok_integer()

int tok_integer	(	lbm_char_channel_t *	chan,
		token_int *	result
	)

Attempt to read an integer value from the character stream.

Parameters

chan	Character channel to read characters from.
res	Result token_int-object if successful.

Returns

A positive value indicating number of characters used if successful. Otherwise a negative status indicator.

tok_match_fixed_size_tokens()

static int tok_match_fixed_size_tokens ( lbm_char_channel_t *  chan, const matcher *  m, unsigned int  start_pos, unsigned int  num, uint32_t *  res  )

static

tok_string()

int tok_string	(	lbm_char_channel_t *	chan,
		unsigned int *	string_len
	)

Attempt to read a string from the character stream. If successful the resulting string is found in tokpar_sym_str.

Parameters

chan	Character channel to read characters from.

Returns

A positive value indicating number of characters used if successful. Otherwise a negative status indicator.

tok_symbol()

int tok_symbol

(

lbm_char_channel_t *

chan

)

Attempt to read a symbol from the character stream. If successful the resulting symbol string is found in tokpar_sym_str.

Parameters

chan	Character channel to read characters from.

Returns

A positive value indicating number of characters used if successful. Otherwise a negative status indicator.

tok_syntax()

int tok_syntax	(	lbm_char_channel_t *	chan,
		uint32_t *	res
	)

Attempt to read a "syntax" token from the character stream. A "syntax" token is for example '(', ')', '@' and so on.

Parameters

chan	Character channel to read characters from.
res	Result token identifier.

Returns

A positive value indicating number of characters used if successful. Otherwise a negative status indicator.

translate_escape_char()

static signed char translate_escape_char ( char  c )

inlinestatic

Variable Documentation

fixed_size_tokens

const matcher fixed_size_tokens[NUM_FIXED_SIZE_TOKENS]

Initial value:

= {
  {"(", TOKOPENPAR, 1},
  {")", TOKCLOSEPAR, 1},
  {"[|", TOKOPENARRAY, 2},
  {"[", TOKOPENBRACK, 1},
  {"]", TOKCLOSEBRACK, 1},
  {".", TOKDOT, 1},
  {"_", TOKDONTCARE, 1},
  {"'", TOKQUOTE, 1},
  {"`", TOKBACKQUOTE, 1},
  {",@", TOKCOMMAAT, 2},
  {",", TOKCOMMA, 1},
  {"?", TOKMATCHANY, 1},
  {"{", TOKOPENCURL, 1},
  {"}", TOKCLOSECURL, 1},
  {"|]", TOKCLOSEARRAY, 2},
  {"@const-start", TOKCONSTSTART, 12},
  {"@const-end", TOKCONSTEND, 10},
}

tokpar_sym_str

char tokpar_sym_str[TOKENIZER_MAX_SYMBOL_AND_STRING_LENGTH+1]

type_qual_table

const matcher type_qual_table[NUM_TYPE_QUALIFIERS]

Initial value:

= {
  {"f64", TOKTYPEF64,  3},
  {"f32", TOKTYPEF32,  3},
  {"i64", TOKTYPEI64,  3},
  {"u64", TOKTYPEU64,  3},
  {"i32", TOKTYPEI32,  3},
  {"u32", TOKTYPEU32,  3},
  {"i"  , TOKTYPEI,    1},
  {"u"  , TOKTYPEU,    1},
  {"b"  , TOKTYPEBYTE, 1}
}