/** * Jason Luu * July 22, 2009 * Tokenizer */ #include #include #include "util.h" #include "token.h" #include "ezxml.h" #include "read_xml_util.h" enum e_token_type GetTokenTypeFromChar(INP enum e_token_type cur_token_type, INP char cur); /* Returns a token list of the text for a given string. */ t_token *GetTokensFromString(INP const char* inString, OUTP int * num_tokens) { const char *cur; t_token * tokens; int i, in_string_index, prev_in_string_index; boolean has_null; enum e_token_type cur_token_type, new_token_type; *num_tokens = i = 0; cur_token_type = TOKEN_NULL; if (inString == NULL) { return NULL; }; cur = inString; /* Count number of tokens */ while (*cur) { new_token_type = GetTokenTypeFromChar(cur_token_type, *cur); if (new_token_type != cur_token_type) { cur_token_type = new_token_type; if (new_token_type != TOKEN_NULL) { i++; } } ++cur; } *num_tokens = i; if (*num_tokens > 0) { tokens = (t_token*)my_calloc(*num_tokens + 1, sizeof(t_token)); } else { return NULL; } /* populate tokens */ i = 0; in_string_index = 0; has_null = TRUE; prev_in_string_index = 0; cur_token_type = TOKEN_NULL; cur = inString; while (*cur) { new_token_type = GetTokenTypeFromChar(cur_token_type, *cur); if (new_token_type != cur_token_type) { if (!has_null) { tokens[i - 1].data[in_string_index - prev_in_string_index] = '\0'; /* NULL the end of the data string */ has_null = TRUE; } if (new_token_type != TOKEN_NULL) { tokens[i].type = new_token_type; tokens[i].data = my_strdup(inString + in_string_index); prev_in_string_index = in_string_index; has_null = FALSE; i++; } cur_token_type = new_token_type; } ++cur; in_string_index++; } assert(i == *num_tokens); tokens[*num_tokens].type = TOKEN_NULL; tokens[*num_tokens].data = NULL; /* Return the list */ return tokens; } void freeTokens(INP t_token *tokens, INP int num_tokens) { int i; for (i = 0; i < num_tokens; i++) { free(tokens[i].data); } free(tokens); } enum e_token_type GetTokenTypeFromChar(INP enum e_token_type cur_token_type, INP char cur) { if (IsWhitespace(cur)) { return TOKEN_NULL; } else { if (cur == '[') { return TOKEN_OPEN_SQUARE_BRACKET; } else if (cur == ']') { return TOKEN_CLOSE_SQUARE_BRACKET; } else if (cur == '{') { return TOKEN_OPEN_SQUIG_BRACKET; } else if (cur == '}') { return TOKEN_CLOSE_SQUIG_BRACKET; } else if (cur == ':') { return TOKEN_COLON; } else if (cur == '.') { return TOKEN_DOT; } else if (cur >= '0' && cur <= '9' && cur_token_type != TOKEN_STRING) { return TOKEN_INT; } else { return TOKEN_STRING; } } } boolean checkTokenType(INP t_token token, OUTP enum e_token_type token_type) { if (token.type != token_type) { return FALSE; } return TRUE; } void my_atof_2D(INOUTP float **matrix, INP int max_i, INP int max_j, INP char *instring) { int i, j; char *cur, *cur2, *copy, *final; copy = my_strdup(instring); final = copy; while (*final != '\0') { final++; } cur = copy; i = j = 0; while (cur != final) { while (IsWhitespace(*cur) && cur != final) { if (j == max_j) { i++; j = 0; } cur++; } if (cur == final) { break; } cur2 = cur; while (!IsWhitespace(*cur2) && cur2 != final) { cur2++; } *cur2 = '\0'; assert(i < max_i && j < max_j); matrix[i][j] = atof(cur); j++; cur = cur2; *cur = ' '; } assert((i == max_i && j == 0) || (i == max_i - 1 && j == max_j)); free(copy); }