/** * This program logs data to a binary file. Functions are included * to convert the binary file to a csv text file. * * Samples are logged at regular intervals. The maximum logging rate * depends on the quality of your SD card and the time required to * read sensor data. This example has been tested at 500 Hz with * good SD card on an Uno. 4000 HZ is possible on a Due. * * If your SD card has a long write latency, it may be necessary to use * slower sample rates. Using a Mega Arduino helps overcome latency * problems since 12 512 byte buffers will be used. * * Data is written to the file using a SD multiple block write command. */ #include #include "SdFat.h" #include "FreeStack.h" #include "UserTypes.h" #ifdef __AVR_ATmega328P__ #include "MinimumSerial.h" MinimumSerial MinSerial; #define Serial MinSerial #endif // __AVR_ATmega328P__ //============================================================================== // Start of configuration constants. //============================================================================== // Abort run on an overrun. Data before the overrun will be saved. #define ABORT_ON_OVERRUN 1 //------------------------------------------------------------------------------ //Interval between data records in microseconds. const uint32_t LOG_INTERVAL_USEC = 2000; //------------------------------------------------------------------------------ // Set USE_SHARED_SPI non-zero for use of an SPI sensor. // May not work for some cards. #ifndef USE_SHARED_SPI #define USE_SHARED_SPI 0 #endif // USE_SHARED_SPI //------------------------------------------------------------------------------ // Pin definitions. // // SD chip select pin. const uint8_t SD_CS_PIN = SS; // // Digital pin to indicate an error, set to -1 if not used. // The led blinks for fatal errors. The led goes on solid for // overrun errors and logging continues unless ABORT_ON_OVERRUN // is non-zero. #ifdef ERROR_LED_PIN #undef ERROR_LED_PIN #endif // ERROR_LED_PIN const int8_t ERROR_LED_PIN = -1; //------------------------------------------------------------------------------ // File definitions. // // Maximum file size in blocks. // The program creates a contiguous file with FILE_BLOCK_COUNT 512 byte blocks. // This file is flash erased using special SD commands. The file will be // truncated if logging is stopped early. const uint32_t FILE_BLOCK_COUNT = 256000; // // log file base name if not defined in UserTypes.h #ifndef FILE_BASE_NAME #define FILE_BASE_NAME "data" #endif // FILE_BASE_NAME //------------------------------------------------------------------------------ // Buffer definitions. // // The logger will use SdFat's buffer plus BUFFER_BLOCK_COUNT-1 additional // buffers. // #ifndef RAMEND // Assume ARM. Use total of ten 512 byte buffers. const uint8_t BUFFER_BLOCK_COUNT = 10; // #elif RAMEND < 0X8FF #error Too little SRAM // #elif RAMEND < 0X10FF // Use total of two 512 byte buffers. const uint8_t BUFFER_BLOCK_COUNT = 2; // #elif RAMEND < 0X20FF // Use total of four 512 byte buffers. const uint8_t BUFFER_BLOCK_COUNT = 4; // #else // RAMEND // Use total of 12 512 byte buffers. const uint8_t BUFFER_BLOCK_COUNT = 12; #endif // RAMEND //============================================================================== // End of configuration constants. //============================================================================== // Temporary log file. Will be deleted if a reset or power failure occurs. #define TMP_FILE_NAME FILE_BASE_NAME "##.bin" // Size of file base name. const uint8_t BASE_NAME_SIZE = sizeof(FILE_BASE_NAME) - 1; const uint8_t FILE_NAME_DIM = BASE_NAME_SIZE + 7; char binName[FILE_NAME_DIM] = FILE_BASE_NAME "00.bin"; SdFat sd; SdBaseFile binFile; // Number of data records in a block. const uint16_t DATA_DIM = (512 - 4)/sizeof(data_t); //Compute fill so block size is 512 bytes. FILL_DIM may be zero. const uint16_t FILL_DIM = 512 - 4 - DATA_DIM*sizeof(data_t); struct block_t { uint16_t count; uint16_t overrun; data_t data[DATA_DIM]; uint8_t fill[FILL_DIM]; }; //============================================================================== // Error messages stored in flash. #define error(msg) {sd.errorPrint(&Serial, F(msg));fatalBlink();} //------------------------------------------------------------------------------ // void fatalBlink() { while (true) { SysCall::yield(); if (ERROR_LED_PIN >= 0) { digitalWrite(ERROR_LED_PIN, HIGH); delay(200); digitalWrite(ERROR_LED_PIN, LOW); delay(200); } } } //------------------------------------------------------------------------------ // read data file and check for overruns void checkOverrun() { bool headerPrinted = false; block_t block; uint32_t bn = 0; if (!binFile.isOpen()) { Serial.println(); Serial.println(F("No current binary file")); return; } binFile.rewind(); Serial.println(); Serial.print(F("FreeStack: ")); Serial.println(FreeStack()); Serial.println(F("Checking overrun errors - type any character to stop")); while (binFile.read(&block, 512) == 512) { if (block.count == 0) { break; } if (block.overrun) { if (!headerPrinted) { Serial.println(); Serial.println(F("Overruns:")); Serial.println(F("fileBlockNumber,sdBlockNumber,overrunCount")); headerPrinted = true; } Serial.print(bn); Serial.print(','); Serial.print(binFile.firstBlock() + bn); Serial.print(','); Serial.println(block.overrun); } bn++; } if (!headerPrinted) { Serial.println(F("No errors found")); } else { Serial.println(F("Done")); } } //----------------------------------------------------------------------------- // Convert binary file to csv file. void binaryToCsv() { uint8_t lastPct = 0; block_t block; uint32_t t0 = millis(); uint32_t syncCluster = 0; SdFile csvFile; char csvName[FILE_NAME_DIM]; if (!binFile.isOpen()) { Serial.println(); Serial.println(F("No current binary file")); return; } Serial.println(); Serial.print(F("FreeStack: ")); Serial.println(FreeStack()); // Create a new csvFile. strcpy(csvName, binName); strcpy(&csvName[BASE_NAME_SIZE + 3], "csv"); if (!csvFile.open(csvName, O_WRITE | O_CREAT | O_TRUNC)) { error("open csvFile failed"); } binFile.rewind(); Serial.print(F("Writing: ")); Serial.print(csvName); Serial.println(F(" - type any character to stop")); printHeader(&csvFile); uint32_t tPct = millis(); while (!Serial.available() && binFile.read(&block, 512) == 512) { uint16_t i; if (block.count == 0 || block.count > DATA_DIM) { break; } if (block.overrun) { csvFile.print(F("OVERRUN,")); csvFile.println(block.overrun); } for (i = 0; i < block.count; i++) { printData(&csvFile, &block.data[i]); } if (csvFile.curCluster() != syncCluster) { csvFile.sync(); syncCluster = csvFile.curCluster(); } if ((millis() - tPct) > 1000) { uint8_t pct = binFile.curPosition()/(binFile.fileSize()/100); if (pct != lastPct) { tPct = millis(); lastPct = pct; Serial.print(pct, DEC); Serial.println('%'); } } if (Serial.available()) { break; } } csvFile.close(); Serial.print(F("Done: ")); Serial.print(0.001*(millis() - t0)); Serial.println(F(" Seconds")); } //----------------------------------------------------------------------------- void createBinFile() { // max number of blocks to erase per erase call const uint32_t ERASE_SIZE = 262144L; uint32_t bgnBlock, endBlock; // Delete old tmp file. if (sd.exists(TMP_FILE_NAME)) { Serial.println(F("Deleting tmp file " TMP_FILE_NAME)); if (!sd.remove(TMP_FILE_NAME)) { error("Can't remove tmp file"); } } // Create new file. Serial.println(F("\nCreating new file")); binFile.close(); if (!binFile.createContiguous(TMP_FILE_NAME, 512 * FILE_BLOCK_COUNT)) { error("createContiguous failed"); } // Get the address of the file on the SD. if (!binFile.contiguousRange(&bgnBlock, &endBlock)) { error("contiguousRange failed"); } // Flash erase all data in the file. Serial.println(F("Erasing all data")); uint32_t bgnErase = bgnBlock; uint32_t endErase; while (bgnErase < endBlock) { endErase = bgnErase + ERASE_SIZE; if (endErase > endBlock) { endErase = endBlock; } if (!sd.card()->erase(bgnErase, endErase)) { error("erase failed"); } bgnErase = endErase + 1; } } //------------------------------------------------------------------------------ // dump data file to Serial void dumpData() { block_t block; if (!binFile.isOpen()) { Serial.println(); Serial.println(F("No current binary file")); return; } binFile.rewind(); Serial.println(); Serial.println(F("Type any character to stop")); delay(1000); printHeader(&Serial); while (!Serial.available() && binFile.read(&block , 512) == 512) { if (block.count == 0) { break; } if (block.overrun) { Serial.print(F("OVERRUN,")); Serial.println(block.overrun); } for (uint16_t i = 0; i < block.count; i++) { printData(&Serial, &block.data[i]); } } Serial.println(F("Done")); } //------------------------------------------------------------------------------ // log data void logData() { createBinFile(); recordBinFile(); renameBinFile(); } //------------------------------------------------------------------------------ void openBinFile() { char name[FILE_NAME_DIM]; strcpy(name, binName); Serial.println(F("\nEnter two digit version")); Serial.write(name, BASE_NAME_SIZE); for (int i = 0; i < 2; i++) { while (!Serial.available()) { SysCall::yield(); } char c = Serial.read(); Serial.write(c); if (c < '0' || c > '9') { Serial.println(F("\nInvalid digit")); return; } name[BASE_NAME_SIZE + i] = c; } Serial.println(&name[BASE_NAME_SIZE+2]); if (!sd.exists(name)) { Serial.println(F("File does not exist")); return; } binFile.close(); strcpy(binName, name); if (!binFile.open(binName, O_READ)) { Serial.println(F("open failed")); return; } Serial.println(F("File opened")); } //------------------------------------------------------------------------------ void recordBinFile() { const uint8_t QUEUE_DIM = BUFFER_BLOCK_COUNT + 1; // Index of last queue location. const uint8_t QUEUE_LAST = QUEUE_DIM - 1; // Allocate extra buffer space. block_t block[BUFFER_BLOCK_COUNT - 1]; block_t* curBlock = 0; block_t* emptyStack[BUFFER_BLOCK_COUNT]; uint8_t emptyTop; uint8_t minTop; block_t* fullQueue[QUEUE_DIM]; uint8_t fullHead = 0; uint8_t fullTail = 0; // Use SdFat's internal buffer. emptyStack[0] = (block_t*)sd.vol()->cacheClear(); if (emptyStack[0] == 0) { error("cacheClear failed"); } // Put rest of buffers on the empty stack. for (int i = 1; i < BUFFER_BLOCK_COUNT; i++) { emptyStack[i] = &block[i - 1]; } emptyTop = BUFFER_BLOCK_COUNT; minTop = BUFFER_BLOCK_COUNT; // Start a multiple block write. if (!sd.card()->writeStart(binFile.firstBlock())) { error("writeStart failed"); } Serial.print(F("FreeStack: ")); Serial.println(FreeStack()); Serial.println(F("Logging - type any character to stop")); bool closeFile = false; uint32_t bn = 0; uint32_t maxLatency = 0; uint32_t overrun = 0; uint32_t overrunTotal = 0; uint32_t logTime = micros(); while(1) { // Time for next data record. logTime += LOG_INTERVAL_USEC; if (Serial.available()) { closeFile = true; } if (closeFile) { if (curBlock != 0) { // Put buffer in full queue. fullQueue[fullHead] = curBlock; fullHead = fullHead < QUEUE_LAST ? fullHead + 1 : 0; curBlock = 0; } } else { if (curBlock == 0 && emptyTop != 0) { curBlock = emptyStack[--emptyTop]; if (emptyTop < minTop) { minTop = emptyTop; } curBlock->count = 0; curBlock->overrun = overrun; overrun = 0; } if ((int32_t)(logTime - micros()) < 0) { error("Rate too fast"); } int32_t delta; do { delta = micros() - logTime; } while (delta < 0); if (curBlock == 0) { overrun++; overrunTotal++; if (ERROR_LED_PIN >= 0) { digitalWrite(ERROR_LED_PIN, HIGH); } #if ABORT_ON_OVERRUN Serial.println(F("Overrun abort")); break; #endif // ABORT_ON_OVERRUN } else { #if USE_SHARED_SPI sd.card()->spiStop(); #endif // USE_SHARED_SPI acquireData(&curBlock->data[curBlock->count++]); #if USE_SHARED_SPI sd.card()->spiStart(); #endif // USE_SHARED_SPI if (curBlock->count == DATA_DIM) { fullQueue[fullHead] = curBlock; fullHead = fullHead < QUEUE_LAST ? fullHead + 1 : 0; curBlock = 0; } } } if (fullHead == fullTail) { // Exit loop if done. if (closeFile) { break; } } else if (!sd.card()->isBusy()) { // Get address of block to write. block_t* pBlock = fullQueue[fullTail]; fullTail = fullTail < QUEUE_LAST ? fullTail + 1 : 0; // Write block to SD. uint32_t usec = micros(); if (!sd.card()->writeData((uint8_t*)pBlock)) { error("write data failed"); } usec = micros() - usec; if (usec > maxLatency) { maxLatency = usec; } // Move block to empty queue. emptyStack[emptyTop++] = pBlock; bn++; if (bn == FILE_BLOCK_COUNT) { // File full so stop break; } } } if (!sd.card()->writeStop()) { error("writeStop failed"); } Serial.print(F("Min Free buffers: ")); Serial.println(minTop); Serial.print(F("Max block write usec: ")); Serial.println(maxLatency); Serial.print(F("Overruns: ")); Serial.println(overrunTotal); // Truncate file if recording stopped early. if (bn != FILE_BLOCK_COUNT) { Serial.println(F("Truncating file")); if (!binFile.truncate(512L * bn)) { error("Can't truncate file"); } } } //------------------------------------------------------------------------------ void recoverTmpFile() { uint16_t count; if (!binFile.open(TMP_FILE_NAME, O_RDWR)) { return; } if (binFile.read(&count, 2) != 2 || count != DATA_DIM) { error("Please delete existing " TMP_FILE_NAME); } Serial.println(F("\nRecovering data in tmp file " TMP_FILE_NAME)); uint32_t bgnBlock = 0; uint32_t endBlock = binFile.fileSize()/512 - 1; // find last used block. while (bgnBlock < endBlock) { uint32_t midBlock = (bgnBlock + endBlock + 1)/2; binFile.seekSet(512*midBlock); if (binFile.read(&count, 2) != 2) error("read"); if (count == 0 || count > DATA_DIM) { endBlock = midBlock - 1; } else { bgnBlock = midBlock; } } // truncate after last used block. if (!binFile.truncate(512*(bgnBlock + 1))) { error("Truncate " TMP_FILE_NAME " failed"); } renameBinFile(); } //----------------------------------------------------------------------------- void renameBinFile() { while (sd.exists(binName)) { if (binName[BASE_NAME_SIZE + 1] != '9') { binName[BASE_NAME_SIZE + 1]++; } else { binName[BASE_NAME_SIZE + 1] = '0'; if (binName[BASE_NAME_SIZE] == '9') { error("Can't create file name"); } binName[BASE_NAME_SIZE]++; } } if (!binFile.rename(sd.vwd(), binName)) { error("Can't rename file"); } Serial.print(F("File renamed: ")); Serial.println(binName); Serial.print(F("File size: ")); Serial.print(binFile.fileSize()/512); Serial.println(F(" blocks")); } //------------------------------------------------------------------------------ void testSensor() { const uint32_t interval = 200000; int32_t diff; data_t data; Serial.println(F("\nTesting - type any character to stop\n")); // Wait for Serial Idle. delay(1000); printHeader(&Serial); uint32_t m = micros(); while (!Serial.available()) { m += interval; do { diff = m - micros(); } while (diff > 0); acquireData(&data); printData(&Serial, &data); } } //------------------------------------------------------------------------------ void setup(void) { if (ERROR_LED_PIN >= 0) { pinMode(ERROR_LED_PIN, OUTPUT); } Serial.begin(9600); // Wait for USB Serial while (!Serial) { SysCall::yield(); } Serial.print(F("\nFreeStack: ")); Serial.println(FreeStack()); Serial.print(F("Records/block: ")); Serial.println(DATA_DIM); if (sizeof(block_t) != 512) { error("Invalid block size"); } // Allow userSetup access to SPI bus. pinMode(SD_CS_PIN, OUTPUT); digitalWrite(SD_CS_PIN, HIGH); // Setup sensors. userSetup(); // Initialize at the highest speed supported by the board that is // not over 50 MHz. Try a lower speed if SPI errors occur. if (!sd.begin(SD_CS_PIN, SD_SCK_MHZ(50))) { sd.initErrorPrint(&Serial); fatalBlink(); } // recover existing tmp file. if (sd.exists(TMP_FILE_NAME)) { Serial.println(F("\nType 'Y' to recover existing tmp file " TMP_FILE_NAME)); while (!Serial.available()) { SysCall::yield(); } if (Serial.read() == 'Y') { recoverTmpFile(); } else { error("'Y' not typed, please manually delete " TMP_FILE_NAME); } } } //------------------------------------------------------------------------------ void loop(void) { // Read any Serial data. do { delay(10); } while (Serial.available() && Serial.read() >= 0); Serial.println(); Serial.println(F("type:")); Serial.println(F("b - open existing bin file")); Serial.println(F("c - convert file to csv")); Serial.println(F("d - dump data to Serial")); Serial.println(F("e - overrun error details")); Serial.println(F("l - list files")); Serial.println(F("r - record data")); Serial.println(F("t - test without logging")); while(!Serial.available()) { SysCall::yield(); } #if WDT_YIELD_TIME_MICROS Serial.println(F("LowLatencyLogger can not run with watchdog timer")); SysCall::halt(); #endif char c = tolower(Serial.read()); // Discard extra Serial data. do { delay(10); } while (Serial.available() && Serial.read() >= 0); if (ERROR_LED_PIN >= 0) { digitalWrite(ERROR_LED_PIN, LOW); } if (c == 'b') { openBinFile(); } else if (c == 'c') { binaryToCsv(); } else if (c == 'd') { dumpData(); } else if (c == 'e') { checkOverrun(); } else if (c == 'l') { Serial.println(F("\nls:")); sd.ls(&Serial, LS_SIZE); } else if (c == 'r') { logData(); } else if (c == 't') { testSensor(); } else { Serial.println(F("Invalid entry")); } }