----------------------------------------------
process_logos.c
----------------------------------------------
/**
 **   process_logos - process Logos files
 ** 
 **  CREATED: 2014.07.21  ABS  copied from col_ed
 ** MODIFIED: 2014.07.24  ABS  added column for inside_something
 ** MODIFIED: 2014.09.07  ABS  got links working, added color to them
 **
 **/

#include        "process_logos.h"

main(argc, argv)
int argc;
char** argv;
{
    /* declare functions */

    int count_commas();
    long long int date_to_epoch();
    char *extract_filename();
    char *replace();

    /* declare local variables */

    char in_o_line[MAX_LINE];
    char in_g_line[MAX_LINE];
    char line_o_array[9][MAX_FIELD];
    char line_g_array[10][MAX_FIELD];
    char prev_device[MAX_LINE];

    int  line_num, stored_line_num, z, sensor_num, current_sensor;
    int this_row;
    int dev, red, grn, blu;
    int inside_something, lucky, lucky2; /* Boolean */
    int sensor_map[NUM_SENSORS];
    double start_times[MAX_O_LINES][NUM_SENSORS + 1];
    double end_times[MAX_O_LINES][NUM_SENSORS + 1];
    int device[MAX_O_LINES][NUM_SENSORS + 1];
    int current_device;
    double time;

    int len;

    double random_double;

    char *tmp_str; /* somebody else mallocs */
    char temp_string[MAX_FIELD];

    FILE* g_fp;
    FILE* l_fp;
    FILE* o_fp;

    /* initialize global variables (including parameters) */

    defaultParms();
    parseArgs(argc, argv);
    ProgName = extract_filename(argv[0]);

    if (Debug != FALSE) {
        fprintf(stderr, "%s: DEBUG: Debug = %d\n", ProgName, Debug);
        fprintf(stderr, "%s: DEBUG: Decimate = %d\n", ProgName, Decimate);
        fprintf(stderr, "%s: DEBUG: Decimate2 = %d\n", ProgName, Decimate2);
        }

    /* initialize local variables */

    o_fp = fopen(O_File, "r");
    g_fp = fopen(G_File, "r");

    l_fp = fopen(L_File, "w");

    srand48(28743.0);

    this_row = 1;

    /* read, process and write csvs */

    /* lat,lon,stime,etime,duration,sensor,trkid,device */

    for (line_num = 0; line_num < NumLinesO; line_num++) {
        len = getl(o_fp, in_o_line, MAX_LINE);
        
        /* strip newline from in_o_line */
        for (z = 0; z < MAX_LINE; z++) {
            if (in_o_line[z] == '\n' || in_o_line[z] == '\r') {
                in_o_line[z] = EOS;
                }
            }

        if (Debug != FALSE) {
            fprintf(stderr, "%s: DEBUG: in_o_line: >%s<\n", ProgName,
                in_o_line);
            }

        if (line_num > 0) {

            /* zero everything first for this line, all sensors */

            for (z = 1; z <= NUM_SENSORS; z++) {
                start_times[line_num][z] = 0.0;
                end_times[line_num][z] = 0.0;
                device[line_num][z] = 0;
                }

            parse_csv_line_to_string(in_o_line, 8, line_o_array);

            if (Debug != FALSE) {
                fprintf(stderr,
                  "%s: DEBUG: line_o_array[3] (start) = %s\n", ProgName,
                  line_o_array[3]);
                fprintf(stderr,
                  "%s: DEBUG: line_o_array[4] (end) = %s\n", ProgName,
                  line_o_array[4]);
                fprintf(stderr,
                  "%s: DEBUG: line_o_array[6] (sensor) = %s\n", ProgName,
                  line_o_array[6]);
                fprintf(stderr,
                  "%s: DEBUG: line_o_array[8] (device) = %s\n", ProgName,
                  line_o_array[8]);
                }

            current_sensor = atoi(line_o_array[6]);
            start_times[line_num][current_sensor] = atof(line_o_array[3]);
            end_times[line_num][current_sensor] = atof(line_o_array[4]);
            device[line_num][current_sensor] = atof(line_o_array[8]);
            }
        }

    if (Debug != FALSE) {
        fprintf(stderr, "%s: DEBUG: --------------------------------------\n",
            ProgName);
        }

    /* lat,lon,alt,vel,dtg,epoch,trkid,dist,device */

    for (line_num = 0; line_num < NumLinesG; line_num++) {

        if (Debug != FALSE) {
            fprintf(stderr, "%s: DEBUG: GPS line_num: %d ================\n",
                ProgName, line_num);
            }

        len = getl(g_fp, in_g_line, MAX_LINE);
        
        /* strip newline from in_g_line */
        for (z = 0; z < MAX_LINE; z++) {
            if (in_g_line[z] == '\n' || in_g_line[z] == '\r') {
                in_g_line[z] = EOS;
                }
            }

        if (Debug != FALSE) {
            fprintf(stderr, "%s: DEBUG: in_g_line = %s\n", ProgName,
                in_g_line);
            }

        if (line_num > 0) {
            parse_csv_line_to_string(in_g_line, 9, line_g_array);

            if (Debug != FALSE) {
                fprintf(stderr,
                  "%s: DEBUG: line_g_array[5] (dtg) = %s\n", ProgName,
                    line_g_array[5]);
                fprintf(stderr,
                  "%s: DEBUG: line_g_array[9] (device) = %s\n", ProgName,
                    line_g_array[9]);
                }

            dev = atoi(line_g_array[9]);
            grn = (int)(255.0*(float)dev/23.0);
            red = 255 - grn;

            if (0 == strcmp(&prev_device[0], &line_g_array[9][0])) {
                fprintf(l_fp, "%d,%d,0.1,%d,%d,255,%d_to_%d;device:%s\n", 
                  this_row - 1, this_row,
                  red, grn,
                  this_row - 1, this_row,
                  &line_g_array[9][0]);
                }

            this_row++;
            strcpy(&prev_device[0], &line_g_array[9][0]);

            time = atof(line_g_array[5]);
            current_device = atof(line_g_array[9]);

            /* compare and write */

            for (sensor_num = 1; sensor_num <= NUM_SENSORS; sensor_num++) {
               sensor_map[sensor_num] = FALSE;
               }
            inside_something = FALSE;

            for (stored_line_num = 0; stored_line_num < NumLinesO;
              stored_line_num++) {
                for (sensor_num = 1; sensor_num <= NUM_SENSORS; sensor_num++) {
                    if (Debug != FALSE) {
                        fprintf(stderr,
                          "%s: DEBUG: stored_line_num: %d; sensor_num: %d; time: %f\n",
                          ProgName, stored_line_num, sensor_num, time);
                        fprintf(stderr,
                          "%s: DEBUG:   start_times: %f; end_times: %f; device: %d\n",
                          ProgName,
                          start_times[stored_line_num][sensor_num],
                          end_times[stored_line_num][sensor_num],
                          device[stored_line_num][sensor_num]);
                        }
                    if (current_device == device[stored_line_num][sensor_num]
                      && time >= start_times[stored_line_num][sensor_num]
                      && time <= end_times[stored_line_num][sensor_num]) {
                        if (Debug != FALSE) {
                            fprintf(stderr,
                              "%s: DEBUG: BINGO: stored_line_num: %d; in sensor_num: %d\n",
                              ProgName, stored_line_num, sensor_num);
                            }
                        sensor_map[sensor_num] = TRUE;
                        inside_something = TRUE;
                        }
                    }

                }

            random_double = drand48();
            if (Debug != FALSE) {
                fprintf(stderr,
                  "%s: DEBUG: random_double: %lf\n", ProgName, random_double);
                }

            lucky = FALSE;
            lucky2 = FALSE;
            if (random_double < 1.0/(double)Decimate) {
                lucky = TRUE;
                if (Debug != FALSE) {
                    fprintf(stderr,
                      "%s: DEBUG: lucky: %d\n", ProgName, lucky);
                    }
                }

            random_double = drand48();
            if (Debug != FALSE) {
                fprintf(stderr,
                  "%s: DEBUG: random_double #2: %lf\n", ProgName, random_double);
                }
            if (random_double < 1.0/(double)Decimate2) {
                lucky2 = TRUE;
                if (Debug != FALSE) {
                    fprintf(stderr,
                      "%s: DEBUG: lucky2: %d\n", ProgName, lucky2);
                    }
                }

            if ((inside_something && lucky2) || (!inside_something && lucky)) {

                /* write a GPS row */

                if (Debug != FALSE) {
                    fprintf(stderr,
                      "%s: DEBUG: writing GPS row\n", ProgName);
                    }

                /* KLUDGE */

                sensor_map[7] = inside_something;

                fprintf(stdout, "%s,", in_g_line);
                for (sensor_num = 1; sensor_num < NUM_SENSORS;
                  sensor_num++) {
                   fprintf(stdout, "%d,", sensor_map[sensor_num]);
                   }

                /* last column different */

                fprintf(stdout, "%d\r\n", sensor_map[NUM_SENSORS]);
                fflush(stdout);
                }
            }
        }

    } /* end main */

/*
 *  parse_csv_line_to_string()
 *
 */
parse_csv_line_to_string(src_line, num_cols, dest_str_array)
char* src_line;
int num_cols;
char dest_str_array[MAX_COLS][MAX_FIELD];
{

    char single_char;
    int col_num = 1;
    int in_char_num = 0;
    int out_char_num = 0;

    if (Debug != FALSE) {
        fprintf(stderr,
          "%s: parse_csv_line_to_string(): DEBUG: src_line: >%s<\n",
          ProgName, src_line);
        }

    while (col_num <= num_cols) {
        single_char = src_line[in_char_num];

        while ((single_char != ',') && (single_char != EOS)
          && (single_char != '\n') && (single_char != '\r')) {
            dest_str_array[col_num][out_char_num++] = single_char;
            single_char = src_line[++in_char_num];
            }
        dest_str_array[col_num][out_char_num] = EOS;

        if (dest_str_array[col_num][0] == EOS) {
            strcpy(&dest_str_array[col_num][0], "333333");
            }

        if (Debug != FALSE) {
            fprintf(stderr,
              "%s: DEBUG: parse_csv_line_to_string: dest_str_array[%d]: >%s<\n",
              ProgName, col_num, dest_str_array[col_num]);
            }

        /* in_char_num now points to first input comma */

#ifdef SUPER_DEBUG
        if (Debug != FALSE) {
            fprintf(stderr,
              "DEBUG: parse_csv_line_to_string(): src_line[%d] = %c (should be comma)\n",
              in_char_num, src_line[in_char_num]);
            }
#endif

        in_char_num++;
        out_char_num = 0;

        col_num++;
        }

    } /* end parse_csv_line_to_string() */

/*
 *  convertToUpper() - convert string to uppercase (from Stackoverflow)
 */
void convertToUpper(char *text, char *nText){

    int col;
    for(col=0; col<=strlen(text); col++){
        if( (text[col] > 96 ) && (text[col] < 123) ) // is the char lower case
            nText[col] = text[col] - 'a' + 'A';   //make upper
        else
            nText[col] = text[col]; //do nothing
    }   
}

/*
 *  defaultParms() - initialize global variables
 */
defaultParms()
{
    Debug = FALSE;
    Decimate = 1;
    Decimate2 = 1;
    NumLinesO = 2;
    NumLinesG = 2;
    strcpy(O_File, "TS14_Collection_Opportunities.csv");
    strcpy(G_File, "TS14_Releasable_GPS.csv");
    strcpy(L_File, "links.txt");

    } /* end defaultParms() */

/*
 *  parseArgs() - parse arguments into global variables
 */
parseArgs(argc, argv)
int argc;
char** argv;
{
    int i, c;

    for (i = 0; i < argc; i++) {
        if (argv[i][0] == '-') {
            switch (argv[i][1]) {
                case 'D' : Debug = TRUE;
                           break;
                case 'd' : Decimate = atoi(&argv[++i][0]);
                           Decimate2 = atoi(&argv[++i][0]);
                           break;
                case 'G' : strcpy(G_File, &argv[++i][0]);
                           break;
                case 'L' : strcpy(L_File, &argv[++i][0]);
                           break;
                case 'O' : strcpy(O_File, &argv[++i][0]);
                           break;
                case 'h' : help(argv);
                           exit(0);
                           break;
                case 'g' : NumLinesG = atoi(&argv[++i][0]);
                           break;
                case 'o' : NumLinesO = atoi(&argv[++i][0]);
                           break;
                case 'u' : usage(argv);
                           exit(0);
                           break;
                default:   fprintf(stderr, "%s: ERROR: illegal flag: %s\n",
                               ProgName, argv[i]);
                           usage(argv);
                           exit(0);
                           break;
                } /* end switch */
            } /* end if */
        } /* end for i */
    } /* end parseArgs() */

/*
 * getl() - modified from Kernighan & Ritchie to take stream
 */
int
getl(stream, s, lim)            /* get line into s, return length */
FILE *stream;
char s[];
int lim;
{
    int c, i;

    i = 0;
    while (--lim > 0 && (c = getc(stream)) != EOF && c != '\n')
        s[i++] = c;
    if (c == '\n')
        s[i++] = c;
    s[i] = '\0';
    return(i);
    } /* end getl() */

/*
 * date_to_epoch()
 */
long long int date_to_epoch(date_string, format_string)
char * date_string;
char * format_string;
{

    struct tm tm;
    time_t t;

    if (strptime(date_string, format_string, &tm) == NULL)

        /* Handle error */
        return(-1);

    tm.tm_isdst = -1;      /* Not set by strptime(); tells mktime()
                              to determine whether daylight saving time
                              is in effect */
    t = mktime(&tm);

    if (t == -1)

        /* Handle error */
        return(-1);

    if (Debug != FALSE) {
        fprintf(stderr,
          "%s: date_to_epoch(): DEBUG: date_string = %s; t = %lld\n",
          ProgName, date_string, (long long int)t);
        }

    return((long long int) t);
    }

/*
 * lines_in_file() - from Stacktrace
 */
int
lines_in_file(filename)
char filename[];
{
    FILE                *fp;            /* */
    int                 c;              /* Nb. int (not char) for the EOF */
    unsigned long       newline_count = 0;

    /* count the newline characters */

    fp = fopen(filename, "r");
    if (fp == NULL) {
        fprintf(stderr, "%s: ERROR: file open of %s failed, error: %s\n",
          ProgName, filename, strerror(errno));
        exit(0);
        }

    while ( (c=fgetc(fp)) != EOF ) {
        if ( c == '\n' ) {
            newline_count++;
            }
        }

    fclose(fp);
    return newline_count;

    } /* end lines_in_file() */

/*
 * extract_filename()
 */
char * extract_filename(char *str)
{
    int     ch = '/';
    size_t  len;
    char   *pdest;
    char   *inpfile = NULL;
 
    // Search backwards for last backslash in filepath
    pdest = strrchr(str, ch);
     
    // if backslash not found in filepath
    if(pdest == NULL )
    {
    printf( "Result:\t%c not found\n", ch );
    pdest = str;  // The whole name is a file in current path?
    }
    else
    {
    pdest++; // Skip the backslash itself.
    }
     
    // extract filename from file path
    len = strlen(pdest);
    inpfile = malloc(len+1);  // Make space for the zero.
    strncpy(inpfile, pdest, len+1);  // Copy including zero.
    return inpfile;
}

/*
 * count_commas()
 */
int count_commas(char *str)
{
    char single_char;

    int char_position = 0;
    int comma_count = 0;

    single_char = str[char_position++];
    while (single_char != EOS) {
        if (single_char == ',') {
            comma_count++;
            }
        single_char = str[char_position++];
        }
    return(comma_count);
    }

/*
 * replace ()
 *
 * from: bytes.com/topic/c/answers/223500-how-replace-substring-string-using-c
 *
 */
#include <stdio.h>
#include <string.h>
#include <stdlib.h>

char *replace(const char *s, const char *old, const char *new)
{
    char *ret;
    int i, count = 0;
    size_t newlen = strlen(new);
    size_t oldlen = strlen(old);

    for (i = 0; s[i] != '\0'; i++) {
        if (strstr(&s[i], old) == &s[i]) {
            count++;
            i += oldlen - 1;
            }
        }

    ret = malloc(i + count * (newlen - oldlen));
    if (ret == NULL)
        exit(EXIT_FAILURE);

    i = 0;
    while (*s) {
        if (strstr(s, old) == s) {
            strcpy(&ret[i], new);
            i += newlen;
            s += oldlen;
            }
        else
            ret[i++] = *s++;
        }
    ret[i] = '\0';

    return ret;
    }

#ifdef UNIT_TEST
int main(void)
{
char mystr[] = "##this is##a examp#le";
char *newstr = NULL;

puts(mystr);
newstr = replace(mystr, "##", "****");
printf("%s\n", newstr);

free(newstr);
return 0;
}
#endif
----------------------------------------------
process_logos.h
----------------------------------------------
/*
 * process_logos.h -- include file for process_logos.c
 */

/* includes */

#include        <stdio.h>
#include        <ctype.h>
#include        <stdlib.h>
#include        <string.h>
#include        <math.h>
#include        <errno.h>
#include        <time.h>

/* constants */

#define TRUE              (1)
#define FALSE             (0)
#define MAX_LINE          (8192)
#define MAX_COLS          (1024)
#define MAX_COMMAND_ROWS  (5000)
#define MAX_FIELD         (64)
#define EOS               (0)
#define STRING_EQUAL      (0)
#define ERROR_CODE        (-1)
#define NUM_SENSORS       (15)
#define MAX_O_LINES       (4444)

/* global variables */

char *ProgName;                /* name of this program -- malloced elsewhere */

/* flag-settable parameters (global variables) */

int Debug;                     /* Boolean: TRUE = debug writes */
int Decimate;
int Decimate2;
int NumLinesO;
int NumLinesG;
char G_File[MAX_LINE];
char O_File[MAX_LINE];
char L_File[MAX_LINE];

/* macros */
----------------------------------------------
Makefile
----------------------------------------------
O_FILES = process_logos.o usage.o help.o
CFLAGS = -lc -g -lm
CC = gcc

process_logos:	$(O_FILES)
	$(CC) $(CFLAGS) -o process_logos $(O_FILES)
----------------------------------------------
Test
----------------------------------------------
#
echo heads
head -4444 ~/Antz2014/Data/Logos/TS14_Collection_Opportunities.csv > O_part.csv
head -3000 ~/Antz2014/Data/Logos/TS14_Releasable_GPS.csv > G_part.csv
#
# I'd love to use -D but it takes forever
#
echo process_logos
/home/Alan/swdev/C/process_logos/process_logos -d 10 10 -o `wc -l O_part.csv` -g `wc -l G_part.csv` -O O_part.csv -G G_part.csv > aggregat.csv
#
echo sed
sed -e 's/_/,/' < aggregat.csv > a2.csv
#
echo cat
cat a2_head.csv a2.csv > temp_test_data.csv
#
echo head
head -3000 temp_test_data.csv > test_data.csv
#
echo R
./Rscript > data_stats.txt
----------------------------------------------