#!/usr/bin/env python

"""
@package mi.dataset.parser.test.test_dosta_abcdjm_dcl
@file marine-integrations/mi/dataset/parser/test/test_dosta_abcdjm_dcl.py
@author <NAME>
@brief Test code for a Dosta_abcdjm_dcl data parser

In the following files, Metadata consists of 4 records
and Garbled consist of 3 records.
There is 1 group of Sensor Data records for each set of metadata.

Files used for testing:

20000101.dosta0.log
  Metadata - 1 set,  Sensor Data - 0 records,  Garbled - 0,  Newline - \n

20010121.dosta1.log
  Metadata - 1 set,  Sensor Data - 21 records,  Garbled - 0,  Newline - \n

20020222.dosta2.log
  Metadata - 2 sets,  Sensor Data - 22 records,  Garbled - 0,  Newline - \r\n

20030314.dosta3.log
  Metadata - 3 sets,  Sensor Data - 14 records,  Garbled - 0,  Newline - \n

20041225.dosta4.log
  Metadata - 2 sets,  Sensor Data - 250 records,  Garbled - 0,  Newline - \n

20050103.dosta5.log
   Metadata - 1 set,  Sensor Data - 3 records,  Garbled - 1,  Newline - \n

20060207.dosta6.log
  Metadata - 2 sets,  Sensor Data - 7 records,  Garbled - 2,  Newline \r\n

20070114.dosta7.log
  This file contains a boatload of invalid sensor data records.  Newline - \r\n
   1. invalid year
   2. invalid month
   3. invalid day
   4. invalid hour
   5. invalid minute
   6. invalid second
   7. invalid product
   8. spaces instead of tabs
   9. a 2-digit serial number
  10. floating point number missing the decimal point
  11. serial number missing
  12. one of the floating point numbers missing
  13. Date in form YYYY-MM-DD
  14. time field missing milliseconds
  15. extra floating point number in sensor data
"""

import unittest
import os
from nose.plugins.attrib import attr

from mi.core.log import get_logger ; log = get_logger()

from mi.dataset.test.test_parser import ParserUnitTestCase

from mi.dataset.parser.dosta_abcdjm_dcl import \
    DostaAbcdjmDclParser, \
    DostaAbcdjmDclRecoveredParser, \
    DostaAbcdjmDclTelemeteredParser, \
    DostaAbcdjmDclRecoveredInstrumentDataParticle, \
    DostaAbcdjmDclTelemeteredInstrumentDataParticle, \
    DostaStateKey

from mi.dataset.dataset_parser import DataSetDriverConfigKeys

from mi.idk.config import Config
RESOURCE_PATH = os.path.join(Config().base_dir(), 'mi', 'dataset', 'driver',
                 'dosta_abcdjm', 'dcl', 'resource')


# Expected tuples for data in file 20010121.dosta1.log
# This file has no sensor data records and should not produce any particles.

# Expected tuples for data in file 20010121.dosta1.log
EXPECTED_20010121_dosta1 = [
    ('2001/01/21 03:06:02.329', '2001', '01', '21', '03', '06', '02', '329', '4831', '2001', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000'),
    ('2001/01/21 03:06:15.908', '2001', '01', '21', '03', '06', '15', '908', '4831', '2001', '25.000', '6.000', '5.000', '4.500', '4.500', '4.500', '4.500', '75.000', '75.000', '12.500'),
    ('2001/01/21 03:06:29.487', '2001', '01', '21', '03', '06', '29', '487', '4831', '2001', '50.000', '12.000', '10.000', '9.000', '9.000', '9.000', '9.000', '150.000', '150.000', '25.000'),
    ('2001/01/21 03:06:43.066', '2001', '01', '21', '03', '06', '43', '066', '4831', '2001', '75.000', '18.000', '15.000', '13.500', '13.500', '13.500', '13.500', '225.000', '225.000', '37.500'),
    ('2001/01/21 03:06:56.645', '2001', '01', '21', '03', '06', '56', '645', '4831', '2001', '100.000', '24.000', '20.000', '18.000', '18.000', '18.000', '18.000', '300.000', '300.000', '50.000'),
    ('2001/01/21 03:07:10.224', '2001', '01', '21', '03', '07', '10', '224', '4831', '2001', '125.000', '30.000', '25.000', '22.500', '22.500', '22.500', '22.500', '375.000', '375.000', '62.500'),
    ('2001/01/21 03:07:23.803', '2001', '01', '21', '03', '07', '23', '803', '4831', '2001', '150.000', '36.000', '30.000', '27.000', '27.000', '27.000', '27.000', '450.000', '450.000', '75.000'),
    ('2001/01/21 03:07:37.382', '2001', '01', '21', '03', '07', '37', '382', '4831', '2001', '175.000', '42.000', '35.000', '31.500', '31.500', '31.500', '31.500', '525.000', '525.000', '87.500'),
    ('2001/01/21 03:07:50.961', '2001', '01', '21', '03', '07', '50', '961', '4831', '2001', '200.000', '48.000', '40.000', '36.000', '36.000', '36.000', '36.000', '600.000', '600.000', '100.000'),
    ('2001/01/21 03:08:04.540', '2001', '01', '21', '03', '08', '04', '540', '4831', '2001', '225.000', '54.000', '45.000', '40.500', '40.500', '40.500', '40.500', '675.000', '675.000', '112.500'),
    ('2001/01/21 03:08:18.119', '2001', '01', '21', '03', '08', '18', '119', '4831', '2001', '250.000', '60.000', '50.000', '45.000', '45.000', '45.000', '45.000', '750.000', '750.000', '125.000'),
    ('2001/01/21 03:08:31.698', '2001', '01', '21', '03', '08', '31', '698', '4831', '2001', '275.000', '66.000', '55.000', '49.500', '49.500', '49.500', '49.500', '825.000', '825.000', '137.500'),
    ('2001/01/21 03:08:45.277', '2001', '01', '21', '03', '08', '45', '277', '4831', '2001', '300.000', '72.000', '60.000', '54.000', '54.000', '54.000', '54.000', '900.000', '900.000', '150.000'),
    ('2001/01/21 03:08:58.856', '2001', '01', '21', '03', '08', '58', '856', '4831', '2001', '325.000', '78.000', '65.000', '58.500', '58.500', '58.500', '58.500', '975.000', '975.000', '162.500'),
    ('2001/01/21 03:09:12.435', '2001', '01', '21', '03', '09', '12', '435', '4831', '2001', '350.000', '84.000', '70.000', '63.000', '63.000', '63.000', '63.000', '1050.000', '1050.000', '175.000'),
    ('2001/01/21 03:09:26.014', '2001', '01', '21', '03', '09', '26', '014', '4831', '2001', '375.000', '90.000', '75.000', '67.500', '67.500', '67.500', '67.500', '1125.000', '1125.000', '187.500'),
    ('2001/01/21 03:09:39.593', '2001', '01', '21', '03', '09', '39', '593', '4831', '2001', '400.000', '96.000', '80.000', '72.000', '72.000', '72.000', '72.000', '1200.000', '1200.000', '200.000'),
    ('2001/01/21 03:09:53.172', '2001', '01', '21', '03', '09', '53', '172', '4831', '2001', '425.000', '102.000', '85.000', '76.500', '76.500', '76.500', '76.500', '1275.000', '1275.000', '212.500'),
    ('2001/01/21 03:10:06.751', '2001', '01', '21', '03', '10', '06', '751', '4831', '2001', '450.000', '108.000', '90.000', '81.000', '81.000', '81.000', '81.000', '1350.000', '1350.000', '225.000'),
    ('2001/01/21 03:10:20.330', '2001', '01', '21', '03', '10', '20', '330', '4831', '2001', '475.000', '114.000', '95.000', '85.500', '85.500', '85.500', '85.500', '1425.000', '1425.000', '237.500'),
    ('2001/01/21 03:10:33.909', '2001', '01', '21', '03', '10', '33', '909', '4831', '2001', '500.000', '120.000', '100.000', '90.000', '90.000', '90.000', '90.000', '1500.000', '1500.000', '250.000'),
]

# Expected tuples for data in file 20020222.dosta2.log
EXPECTED_20020222_dosta2 = [
    ('2002/02/22 05:08:04.331', '2002', '02', '22', '05', '08', '04', '331', '4831', '2002', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000'),
    ('2002/02/22 05:08:17.910', '2002', '02', '22', '05', '08', '17', '910', '4831', '2002', '11.628', '2.791', '2.326', '2.093', '2.093', '2.093', '2.093', '34.884', '34.884', '5.814'),
    ('2002/02/22 05:08:31.489', '2002', '02', '22', '05', '08', '31', '489', '4831', '2002', '23.256', '5.581', '4.651', '4.186', '4.186', '4.186', '4.186', '69.767', '69.767', '11.628'),
    ('2002/02/22 05:08:45.068', '2002', '02', '22', '05', '08', '45', '068', '4831', '2002', '34.884', '8.372', '6.977', '6.279', '6.279', '6.279', '6.279', '104.651', '104.651', '17.442'),
    ('2002/02/22 05:08:58.647', '2002', '02', '22', '05', '08', '58', '647', '4831', '2002', '46.512', '11.163', '9.302', '8.372', '8.372', '8.372', '8.372', '139.535', '139.535', '23.256'),
    ('2002/02/22 05:09:12.226', '2002', '02', '22', '05', '09', '12', '226', '4831', '2002', '58.140', '13.953', '11.628', '10.465', '10.465', '10.465', '10.465', '174.419', '174.419', '29.070'),
    ('2002/02/22 05:09:25.805', '2002', '02', '22', '05', '09', '25', '805', '4831', '2002', '69.767', '16.744', '13.953', '12.558', '12.558', '12.558', '12.558', '209.302', '209.302', '34.884'),
    ('2002/02/22 05:09:39.384', '2002', '02', '22', '05', '09', '39', '384', '4831', '2002', '81.395', '19.535', '16.279', '14.651', '14.651', '14.651', '14.651', '244.186', '244.186', '40.698'),
    ('2002/02/22 05:09:52.963', '2002', '02', '22', '05', '09', '52', '963', '4831', '2002', '93.023', '22.326', '18.605', '16.744', '16.744', '16.744', '16.744', '279.070', '279.070', '46.512'),
    ('2002/02/22 05:10:06.542', '2002', '02', '22', '05', '10', '06', '542', '4831', '2002', '104.651', '25.116', '20.930', '18.837', '18.837', '18.837', '18.837', '313.953', '313.953', '52.326'),
    ('2002/02/22 05:10:20.121', '2002', '02', '22', '05', '10', '20', '121', '4831', '2002', '116.279', '27.907', '23.256', '20.930', '20.930', '20.930', '20.930', '348.837', '348.837', '58.140'),
    ('2002/02/22 05:10:33.700', '2002', '02', '22', '05', '10', '33', '700', '4831', '2002', '127.907', '30.698', '25.581', '23.023', '23.023', '23.023', '23.023', '383.721', '383.721', '63.953'),
    ('2002/02/22 05:10:47.279', '2002', '02', '22', '05', '10', '47', '279', '4831', '2002', '139.535', '33.488', '27.907', '25.116', '25.116', '25.116', '25.116', '418.605', '418.605', '69.767'),
    ('2002/02/22 05:11:00.858', '2002', '02', '22', '05', '11', '00', '858', '4831', '2002', '151.163', '36.279', '30.233', '27.209', '27.209', '27.209', '27.209', '453.488', '453.488', '75.581'),
    ('2002/02/22 05:11:14.437', '2002', '02', '22', '05', '11', '14', '437', '4831', '2002', '162.791', '39.070', '32.558', '29.302', '29.302', '29.302', '29.302', '488.372', '488.372', '81.395'),
    ('2002/02/22 05:11:28.016', '2002', '02', '22', '05', '11', '28', '016', '4831', '2002', '174.419', '41.860', '34.884', '31.395', '31.395', '31.395', '31.395', '523.256', '523.256', '87.209'),
    ('2002/02/22 05:11:41.595', '2002', '02', '22', '05', '11', '41', '595', '4831', '2002', '186.047', '44.651', '37.209', '33.488', '33.488', '33.488', '33.488', '558.140', '558.140', '93.023'),
    ('2002/02/22 05:11:55.174', '2002', '02', '22', '05', '11', '55', '174', '4831', '2002', '197.674', '47.442', '39.535', '35.581', '35.581', '35.581', '35.581', '593.023', '593.023', '98.837'),
    ('2002/02/22 05:12:08.753', '2002', '02', '22', '05', '12', '08', '753', '4831', '2002', '209.302', '50.233', '41.860', '37.674', '37.674', '37.674', '37.674', '627.907', '627.907', '104.651'),
    ('2002/02/22 05:12:22.332', '2002', '02', '22', '05', '12', '22', '332', '4831', '2002', '220.930', '53.023', '44.186', '39.767', '39.767', '39.767', '39.767', '662.791', '662.791', '110.465'),
    ('2002/02/22 05:12:35.911', '2002', '02', '22', '05', '12', '35', '911', '4831', '2002', '232.558', '55.814', '46.512', '41.860', '41.860', '41.860', '41.860', '697.674', '697.674', '116.279'),
    ('2002/02/22 05:12:49.490', '2002', '02', '22', '05', '12', '49', '490', '4831', '2002', '244.186', '58.605', '48.837', '43.953', '43.953', '43.953', '43.953', '732.558', '732.558', '122.093'),
    ('2002/02/22 05:13:57.385', '2002', '02', '22', '05', '13', '57', '385', '4831', '2002', '255.814', '61.395', '51.163', '46.047', '46.047', '46.047', '46.047', '767.442', '767.442', '127.907'),
    ('2002/02/22 05:14:10.964', '2002', '02', '22', '05', '14', '10', '964', '4831', '2002', '267.442', '64.186', '53.488', '48.140', '48.140', '48.140', '48.140', '802.326', '802.326', '133.721'),
    ('2002/02/22 05:14:24.543', '2002', '02', '22', '05', '14', '24', '543', '4831', '2002', '279.070', '66.977', '55.814', '50.233', '50.233', '50.233', '50.233', '837.209', '837.209', '139.535'),
    ('2002/02/22 05:14:38.122', '2002', '02', '22', '05', '14', '38', '122', '4831', '2002', '290.698', '69.767', '58.140', '52.326', '52.326', '52.326', '52.326', '872.093', '872.093', '145.349'),
    ('2002/02/22 05:14:51.701', '2002', '02', '22', '05', '14', '51', '701', '4831', '2002', '302.326', '72.558', '60.465', '54.419', '54.419', '54.419', '54.419', '906.977', '906.977', '151.163'),
    ('2002/02/22 05:15:05.280', '2002', '02', '22', '05', '15', '05', '280', '4831', '2002', '313.953', '75.349', '62.791', '56.512', '56.512', '56.512', '56.512', '941.860', '941.860', '156.977'),
    ('2002/02/22 05:15:18.859', '2002', '02', '22', '05', '15', '18', '859', '4831', '2002', '325.581', '78.140', '65.116', '58.605', '58.605', '58.605', '58.605', '976.744', '976.744', '162.791'),
    ('2002/02/22 05:15:32.438', '2002', '02', '22', '05', '15', '32', '438', '4831', '2002', '337.209', '80.930', '67.442', '60.698', '60.698', '60.698', '60.698', '1011.628', '1011.628', '168.605'),
    ('2002/02/22 05:15:46.017', '2002', '02', '22', '05', '15', '46', '017', '4831', '2002', '348.837', '83.721', '69.767', '62.791', '62.791', '62.791', '62.791', '1046.512', '1046.512', '174.419'),
    ('2002/02/22 05:15:59.596', '2002', '02', '22', '05', '15', '59', '596', '4831', '2002', '360.465', '86.512', '72.093', '64.884', '64.884', '64.884', '64.884', '1081.395', '1081.395', '180.233'),
    ('2002/02/22 05:16:13.175', '2002', '02', '22', '05', '16', '13', '175', '4831', '2002', '372.093', '89.302', '74.419', '66.977', '66.977', '66.977', '66.977', '1116.279', '1116.279', '186.047'),
    ('2002/02/22 05:16:26.754', '2002', '02', '22', '05', '16', '26', '754', '4831', '2002', '383.721', '92.093', '76.744', '69.070', '69.070', '69.070', '69.070', '1151.163', '1151.163', '191.860'),
    ('2002/02/22 05:16:40.333', '2002', '02', '22', '05', '16', '40', '333', '4831', '2002', '395.349', '94.884', '79.070', '71.163', '71.163', '71.163', '71.163', '1186.047', '1186.047', '197.674'),
    ('2002/02/22 05:16:53.912', '2002', '02', '22', '05', '16', '53', '912', '4831', '2002', '406.977', '97.674', '81.395', '73.256', '73.256', '73.256', '73.256', '1220.930', '1220.930', '203.488'),
    ('2002/02/22 05:17:07.491', '2002', '02', '22', '05', '17', '07', '491', '4831', '2002', '418.605', '100.465', '83.721', '75.349', '75.349', '75.349', '75.349', '1255.814', '1255.814', '209.302'),
    ('2002/02/22 05:17:21.070', '2002', '02', '22', '05', '17', '21', '070', '4831', '2002', '430.233', '103.256', '86.047', '77.442', '77.442', '77.442', '77.442', '1290.698', '1290.698', '215.116'),
    ('2002/02/22 05:17:34.649', '2002', '02', '22', '05', '17', '34', '649', '4831', '2002', '441.860', '106.047', '88.372', '79.535', '79.535', '79.535', '79.535', '1325.581', '1325.581', '220.930'),
    ('2002/02/22 05:17:48.228', '2002', '02', '22', '05', '17', '48', '228', '4831', '2002', '453.488', '108.837', '90.698', '81.628', '81.628', '81.628', '81.628', '1360.465', '1360.465', '226.744'),
    ('2002/02/22 05:18:01.807', '2002', '02', '22', '05', '18', '01', '807', '4831', '2002', '465.116', '111.628', '93.023', '83.721', '83.721', '83.721', '83.721', '1395.349', '1395.349', '232.558'),
    ('2002/02/22 05:18:15.386', '2002', '02', '22', '05', '18', '15', '386', '4831', '2002', '476.744', '114.419', '95.349', '85.814', '85.814', '85.814', '85.814', '1430.233', '1430.233', '238.372'),
    ('2002/02/22 05:18:28.965', '2002', '02', '22', '05', '18', '28', '965', '4831', '2002', '488.372', '117.209', '97.674', '87.907', '87.907', '87.907', '87.907', '1465.116', '1465.116', '244.186'),
    ('2002/02/22 05:18:42.544', '2002', '02', '22', '05', '18', '42', '544', '4831', '2002', '500.000', '120.000', '100.000', '90.000', '90.000', '90.000', '90.000', '1500.000', '1500.000', '250.000'),
]

# Expected tuples for data in file 20030314.dosta3.log
EXPECTED_20030314_dosta3 = [
    ('2003/03/14 07:10:06.333', '2003', '03', '14', '07', '10', '06', '333', '4831', '2003', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000'),
    ('2003/03/14 07:10:19.912', '2003', '03', '14', '07', '10', '19', '912', '4831', '2003', '12.195', '2.927', '2.439', '2.195', '2.195', '2.195', '2.195', '36.585', '36.585', '6.098'),
    ('2003/03/14 07:10:33.491', '2003', '03', '14', '07', '10', '33', '491', '4831', '2003', '24.390', '5.854', '4.878', '4.390', '4.390', '4.390', '4.390', '73.171', '73.171', '12.195'),
    ('2003/03/14 07:10:47.070', '2003', '03', '14', '07', '10', '47', '070', '4831', '2003', '36.585', '8.780', '7.317', '6.585', '6.585', '6.585', '6.585', '109.756', '109.756', '18.293'),
    ('2003/03/14 07:11:00.649', '2003', '03', '14', '07', '11', '00', '649', '4831', '2003', '48.780', '11.707', '9.756', '8.780', '8.780', '8.780', '8.780', '146.341', '146.341', '24.390'),
    ('2003/03/14 07:11:14.228', '2003', '03', '14', '07', '11', '14', '228', '4831', '2003', '60.976', '14.634', '12.195', '10.976', '10.976', '10.976', '10.976', '182.927', '182.927', '30.488'),
    ('2003/03/14 07:11:27.807', '2003', '03', '14', '07', '11', '27', '807', '4831', '2003', '73.171', '17.561', '14.634', '13.171', '13.171', '13.171', '13.171', '219.512', '219.512', '36.585'),
    ('2003/03/14 07:11:41.386', '2003', '03', '14', '07', '11', '41', '386', '4831', '2003', '85.366', '20.488', '17.073', '15.366', '15.366', '15.366', '15.366', '256.098', '256.098', '42.683'),
    ('2003/03/14 07:11:54.965', '2003', '03', '14', '07', '11', '54', '965', '4831', '2003', '97.561', '23.415', '19.512', '17.561', '17.561', '17.561', '17.561', '292.683', '292.683', '48.780'),
    ('2003/03/14 07:12:08.544', '2003', '03', '14', '07', '12', '08', '544', '4831', '2003', '109.756', '26.341', '21.951', '19.756', '19.756', '19.756', '19.756', '329.268', '329.268', '54.878'),
    ('2003/03/14 07:12:22.123', '2003', '03', '14', '07', '12', '22', '123', '4831', '2003', '121.951', '29.268', '24.390', '21.951', '21.951', '21.951', '21.951', '365.854', '365.854', '60.976'),
    ('2003/03/14 07:12:35.702', '2003', '03', '14', '07', '12', '35', '702', '4831', '2003', '134.146', '32.195', '26.829', '24.146', '24.146', '24.146', '24.146', '402.439', '402.439', '67.073'),
    ('2003/03/14 07:12:49.281', '2003', '03', '14', '07', '12', '49', '281', '4831', '2003', '146.341', '35.122', '29.268', '26.341', '26.341', '26.341', '26.341', '439.024', '439.024', '73.171'),
    ('2003/03/14 07:13:02.860', '2003', '03', '14', '07', '13', '02', '860', '4831', '2003', '158.537', '38.049', '31.707', '28.537', '28.537', '28.537', '28.537', '475.610', '475.610', '79.268'),
    ('2003/03/14 07:14:10.755', '2003', '03', '14', '07', '14', '10', '755', '4831', '2003', '170.732', '40.976', '34.146', '30.732', '30.732', '30.732', '30.732', '512.195', '512.195', '85.366'),
    ('2003/03/14 07:14:24.334', '2003', '03', '14', '07', '14', '24', '334', '4831', '2003', '182.927', '43.902', '36.585', '32.927', '32.927', '32.927', '32.927', '548.780', '548.780', '91.463'),
    ('2003/03/14 07:14:37.913', '2003', '03', '14', '07', '14', '37', '913', '4831', '2003', '195.122', '46.829', '39.024', '35.122', '35.122', '35.122', '35.122', '585.366', '585.366', '97.561'),
    ('2003/03/14 07:14:51.492', '2003', '03', '14', '07', '14', '51', '492', '4831', '2003', '207.317', '49.756', '41.463', '37.317', '37.317', '37.317', '37.317', '621.951', '621.951', '103.659'),
    ('2003/03/14 07:15:05.071', '2003', '03', '14', '07', '15', '05', '071', '4831', '2003', '219.512', '52.683', '43.902', '39.512', '39.512', '39.512', '39.512', '658.537', '658.537', '109.756'),
    ('2003/03/14 07:15:18.650', '2003', '03', '14', '07', '15', '18', '650', '4831', '2003', '231.707', '55.610', '46.341', '41.707', '41.707', '41.707', '41.707', '695.122', '695.122', '115.854'),
    ('2003/03/14 07:15:32.229', '2003', '03', '14', '07', '15', '32', '229', '4831', '2003', '243.902', '58.537', '48.780', '43.902', '43.902', '43.902', '43.902', '731.707', '731.707', '121.951'),
    ('2003/03/14 07:15:45.808', '2003', '03', '14', '07', '15', '45', '808', '4831', '2003', '256.098', '61.463', '51.220', '46.098', '46.098', '46.098', '46.098', '768.293', '768.293', '128.049'),
    ('2003/03/14 07:15:59.387', '2003', '03', '14', '07', '15', '59', '387', '4831', '2003', '268.293', '64.390', '53.659', '48.293', '48.293', '48.293', '48.293', '804.878', '804.878', '134.146'),
    ('2003/03/14 07:16:12.966', '2003', '03', '14', '07', '16', '12', '966', '4831', '2003', '280.488', '67.317', '56.098', '50.488', '50.488', '50.488', '50.488', '841.463', '841.463', '140.244'),
    ('2003/03/14 07:16:26.545', '2003', '03', '14', '07', '16', '26', '545', '4831', '2003', '292.683', '70.244', '58.537', '52.683', '52.683', '52.683', '52.683', '878.049', '878.049', '146.341'),
    ('2003/03/14 07:16:40.124', '2003', '03', '14', '07', '16', '40', '124', '4831', '2003', '304.878', '73.171', '60.976', '54.878', '54.878', '54.878', '54.878', '914.634', '914.634', '152.439'),
    ('2003/03/14 07:16:53.703', '2003', '03', '14', '07', '16', '53', '703', '4831', '2003', '317.073', '76.098', '63.415', '57.073', '57.073', '57.073', '57.073', '951.220', '951.220', '158.537'),
    ('2003/03/14 07:17:07.282', '2003', '03', '14', '07', '17', '07', '282', '4831', '2003', '329.268', '79.024', '65.854', '59.268', '59.268', '59.268', '59.268', '987.805', '987.805', '164.634'),
    ('2003/03/14 07:18:15.177', '2003', '03', '14', '07', '18', '15', '177', '4831', '2003', '341.463', '81.951', '68.293', '61.463', '61.463', '61.463', '61.463', '1024.390', '1024.390', '170.732'),
    ('2003/03/14 07:18:28.756', '2003', '03', '14', '07', '18', '28', '756', '4831', '2003', '353.659', '84.878', '70.732', '63.659', '63.659', '63.659', '63.659', '1060.976', '1060.976', '176.829'),
    ('2003/03/14 07:18:42.335', '2003', '03', '14', '07', '18', '42', '335', '4831', '2003', '365.854', '87.805', '73.171', '65.854', '65.854', '65.854', '65.854', '1097.561', '1097.561', '182.927'),
    ('2003/03/14 07:18:55.914', '2003', '03', '14', '07', '18', '55', '914', '4831', '2003', '378.049', '90.732', '75.610', '68.049', '68.049', '68.049', '68.049', '1134.146', '1134.146', '189.024'),
    ('2003/03/14 07:19:09.493', '2003', '03', '14', '07', '19', '09', '493', '4831', '2003', '390.244', '93.659', '78.049', '70.244', '70.244', '70.244', '70.244', '1170.732', '1170.732', '195.122'),
    ('2003/03/14 07:19:23.072', '2003', '03', '14', '07', '19', '23', '072', '4831', '2003', '402.439', '96.585', '80.488', '72.439', '72.439', '72.439', '72.439', '1207.317', '1207.317', '201.220'),
    ('2003/03/14 07:19:36.651', '2003', '03', '14', '07', '19', '36', '651', '4831', '2003', '414.634', '99.512', '82.927', '74.634', '74.634', '74.634', '74.634', '1243.902', '1243.902', '207.317'),
    ('2003/03/14 07:19:50.230', '2003', '03', '14', '07', '19', '50', '230', '4831', '2003', '426.829', '102.439', '85.366', '76.829', '76.829', '76.829', '76.829', '1280.488', '1280.488', '213.415'),
    ('2003/03/14 07:20:03.809', '2003', '03', '14', '07', '20', '03', '809', '4831', '2003', '439.024', '105.366', '87.805', '79.024', '79.024', '79.024', '79.024', '1317.073', '1317.073', '219.512'),
    ('2003/03/14 07:20:17.388', '2003', '03', '14', '07', '20', '17', '388', '4831', '2003', '451.220', '108.293', '90.244', '81.220', '81.220', '81.220', '81.220', '1353.659', '1353.659', '225.610'),
    ('2003/03/14 07:20:30.967', '2003', '03', '14', '07', '20', '30', '967', '4831', '2003', '463.415', '111.220', '92.683', '83.415', '83.415', '83.415', '83.415', '1390.244', '1390.244', '231.707'),
    ('2003/03/14 07:20:44.546', '2003', '03', '14', '07', '20', '44', '546', '4831', '2003', '475.610', '114.146', '95.122', '85.610', '85.610', '85.610', '85.610', '1426.829', '1426.829', '237.805'),
    ('2003/03/14 07:20:58.125', '2003', '03', '14', '07', '20', '58', '125', '4831', '2003', '487.805', '117.073', '97.561', '87.805', '87.805', '87.805', '87.805', '1463.415', '1463.415', '243.902'),
    ('2003/03/14 07:21:11.704', '2003', '03', '14', '07', '21', '11', '704', '4831', '2003', '500.000', '120.000', '100.000', '90.000', '90.000', '90.000', '90.000', '1500.000', '1500.000', '250.000'),
]

# Expected tuples for data in file 20041225.dosta4.log
# This file produces 500 particles.  It will be used for the large import test only.
# No verification of particles from this file are done in unit test.
# Particle results will be verified in integration and qualification tests.

# Expected tuples for data in file 20050103.dosta5.log
EXPECTED_20050103_dosta5 = [
    ('2005/01/03 11:14:10.337', '2005', '01', '03', '11', '14', '10', '337', '4831', '205', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000'),
    ('2005/01/03 11:14:23.916', '2005', '01', '03', '11', '14', '23', '916', '4831', '205', '250.000', '60.000', '50.000', '45.000', '45.000', '45.000', '45.000', '750.000', '750.000', '125.000'),
    ('2005/01/03 11:14:37.495', '2005', '01', '03', '11', '14', '37', '495', '4831', '205', '500.000', '120.000', '100.000', '90.000', '90.000', '90.000', '90.000', '1500.000', '1500.000', '250.000'),
]

# Expected tuples for data in file 20060207.dosta6.log
EXPECTED_20060207_dosta6 = [
    ('2006/02/07 13:16:12.339', '2006', '02', '07', '13', '16', '12', '339', '4831', '2006', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000'),
    ('2006/02/07 13:16:25.918', '2006', '02', '07', '13', '16', '25', '918', '4831', '2006', '38.462', '9.231', '7.692', '6.923', '6.923', '6.923', '6.923', '115.385', '115.385', '19.231'),
    ('2006/02/07 13:16:39.497', '2006', '02', '07', '13', '16', '39', '497', '4831', '2006', '76.923', '18.462', '15.385', '13.846', '13.846', '13.846', '13.846', '230.769', '230.769', '38.462'),
    ('2006/02/07 13:16:53.076', '2006', '02', '07', '13', '16', '53', '076', '4831', '2006', '115.385', '27.692', '23.077', '20.769', '20.769', '20.769', '20.769', '346.154', '346.154', '57.692'),
    ('2006/02/07 13:17:06.655', '2006', '02', '07', '13', '17', '06', '655', '4831', '2006', '153.846', '36.923', '30.769', '27.692', '27.692', '27.692', '27.692', '461.538', '461.538', '76.923'),
    ('2006/02/07 13:17:20.234', '2006', '02', '07', '13', '17', '20', '234', '4831', '2006', '192.308', '46.154', '38.462', '34.615', '34.615', '34.615', '34.615', '576.923', '576.923', '96.154'),
    ('2006/02/07 13:17:33.813', '2006', '02', '07', '13', '17', '33', '813', '4831', '2006', '230.769', '55.385', '46.154', '41.538', '41.538', '41.538', '41.538', '692.308', '692.308', '115.385'),
    ('2006/02/07 13:19:22.445', '2006', '02', '07', '13', '19', '22', '445', '4831', '2006', '269.231', '64.615', '53.846', '48.462', '48.462', '48.462', '48.462', '807.692', '807.692', '134.615'),
    ('2006/02/07 13:19:36.024', '2006', '02', '07', '13', '19', '36', '024', '4831', '2006', '307.692', '73.846', '61.538', '55.385', '55.385', '55.385', '55.385', '923.077', '923.077', '153.846'),
    ('2006/02/07 13:19:49.603', '2006', '02', '07', '13', '19', '49', '603', '4831', '2006', '346.154', '83.077', '69.231', '62.308', '62.308', '62.308', '62.308', '1038.462', '1038.462', '173.077'),
    ('2006/02/07 13:20:03.182', '2006', '02', '07', '13', '20', '03', '182', '4831', '2006', '384.615', '92.308', '76.923', '69.231', '69.231', '69.231', '69.231', '1153.846', '1153.846', '192.308'),
    ('2006/02/07 13:20:16.761', '2006', '02', '07', '13', '20', '16', '761', '4831', '2006', '423.077', '101.538', '84.615', '76.154', '76.154', '76.154', '76.154', '1269.231', '1269.231', '211.538'),
    ('2006/02/07 13:20:30.340', '2006', '02', '07', '13', '20', '30', '340', '4831', '2006', '461.538', '110.769', '92.308', '83.077', '83.077', '83.077', '83.077', '1384.615', '1384.615', '230.769'),
    ('2006/02/07 13:20:43.919', '2006', '02', '07', '13', '20', '43', '919', '4831', '2006', '500.000', '120.000', '100.000', '90.000', '90.000', '90.000', '90.000', '1500.000', '1500.000', '250.000'),
]

FILE0 = '20000101.dosta0.log'
FILE1 = '20010121.dosta1.log'
FILE2 = '20020222.dosta2.log'
FILE3 = '20030314.dosta3.log'
FILE4 = '20041225.dosta4.log'
FILE5 = '20050103.dosta5.log'
FILE6 = '20060207.dosta6.log'
FILE7 = '20070114.dosta7.log'

EXPECTED_FILE1 = EXPECTED_20010121_dosta1
EXPECTED_FILE2 = EXPECTED_20020222_dosta2
EXPECTED_FILE3 = EXPECTED_20030314_dosta3
EXPECTED_FILE5 = EXPECTED_20050103_dosta5
EXPECTED_FILE6 = EXPECTED_20060207_dosta6


@attr('UNIT', group='mi')
class DostaAbcdjmDclParserUnitTestCase(ParserUnitTestCase):

    def create_rec_parser(self, file_handle, new_state=None):
        """
        This function creates a DostaAbcdjmDcl parser for recovered data.
        """
        parser = DostaAbcdjmDclRecoveredParser(self.rec_config,
            file_handle, new_state, self.rec_state_callback,
            self.rec_pub_callback, self.rec_exception_callback)
        return parser

    def create_tel_parser(self, file_handle, new_state=None):
        """
        This function creates a DostaAbcdjmDcl parser for telemetered data.
        """
        parser = DostaAbcdjmDclTelemeteredParser(self.tel_config,
            file_handle, new_state, self.rec_state_callback,
            self.tel_pub_callback, self.tel_exception_callback)
        return parser

    def open_file(self, filename):
        file = open(os.path.join(RESOURCE_PATH, filename), mode='r')
        return file

    def rec_state_callback(self, state, file_ingested):
        """ Call back method to watch what comes in via the position callback """
        self.rec_state_callback_value = state
        self.rec_file_ingested_value = file_ingested

    def tel_state_callback(self, state, file_ingested):
        """ Call back method to watch what comes in via the position callback """
        self.tel_state_callback_value = state
        self.tel_file_ingested_value = file_ingested

    def rec_pub_callback(self, pub):
        """ Call back method to watch what comes in via the publish callback """
        self.rec_publish_callback_value = pub

    def tel_pub_callback(self, pub):
        """ Call back method to watch what comes in via the publish callback """
        self.tel_publish_callback_value = pub

    def rec_exception_callback(self, exception):
        """ Call back method to watch what comes in via the exception callback """
        self.rec_exception_callback_value = exception
        self.rec_exceptions_detected += 1

    def tel_exception_callback(self, exception):
        """ Call back method to watch what comes in via the exception callback """
        self.tel_exception_callback_value = exception
        self.tel_exceptions_detected += 1

    def setUp(self):
        ParserUnitTestCase.setUp(self)

        self.rec_config = {
            DataSetDriverConfigKeys.PARTICLE_MODULE:
                'mi.dataset.parser.dosta_abcdjm_dcl',
            DataSetDriverConfigKeys.PARTICLE_CLASS:
                None
        }

        self.tel_config = {
            DataSetDriverConfigKeys.PARTICLE_MODULE:
                'mi.dataset.parser.dosta_abcdjm_dcl',
            DataSetDriverConfigKeys.PARTICLE_CLASS:
                None
        }

        self.rec_state_callback_value = None
        self.rec_file_ingested_value = False
        self.rec_publish_callback_value = None
        self.rec_exception_callback_value = None
        self.rec_exceptions_detected = 0

        self.tel_state_callback_value = None
        self.tel_file_ingested_value = False
        self.tel_publish_callback_value = None
        self.tel_exception_callback_value = None
        self.tel_exceptions_detected = 0

        self.maxDiff = None

    def test_big_giant_input(self):
        """
        Read a large file and verify that all expected particles can be read.
        Verification is not done at this time, but will be done during
        integration and qualification testing.
        File used for this test has 500 total particles.
        """
        log.debug('===== START TEST BIG GIANT INPUT RECOVERED =====')
        in_file = self.open_file(FILE4)
        parser = self.create_rec_parser(in_file)

        number_expected_results = 500

        # In a single read, get all particles in this file.
        result = parser.get_records(number_expected_results)
        self.assertEqual(len(result), number_expected_results)

        in_file.close()
        self.assertEqual(self.rec_exception_callback_value, None)

        log.debug('===== START TEST BIG GIANT INPUT TELEMETERED =====')
        in_file = self.open_file(FILE4)
        parser = self.create_tel_parser(in_file)

        # In a single read, get all particles in this file.
        result = parser.get_records(number_expected_results)
        self.assertEqual(len(result), number_expected_results)

        in_file.close()
        self.assertEqual(self.tel_exception_callback_value, None)

        log.debug('===== END TEST BIG GIANT INPUT =====')

    def test_get_many(self):
        """
        Read a file and pull out multiple data particles at one time.
        Verify that the results are those we expected.
        """
        log.debug('===== START TEST GET MANY RECOVERED =====')
        in_file = self.open_file(FILE3)
        parser = self.create_rec_parser(in_file)

        # Generate a list of expected result particles.
        expected_particle = []
        for expected in EXPECTED_FILE3:
            particle = DostaAbcdjmDclRecoveredInstrumentDataParticle(expected)
            expected_particle.append(particle)

        # In a single read, get all particles for this file.
        result = parser.get_records(len(expected_particle))
        self.assertEqual(result, expected_particle)

        self.assertEqual(self.rec_exception_callback_value, None)
        in_file.close()

        log.debug('===== START TEST GET MANY TELEMETERED =====')
        in_file = self.open_file(FILE3)
        parser = self.create_tel_parser(in_file)

        # Generate a list of expected result particles.
        expected_particle = []
        for expected in EXPECTED_FILE3:
            particle = DostaAbcdjmDclTelemeteredInstrumentDataParticle(expected)
            expected_particle.append(particle)

        # In a single read, get all particles for this file.
        result = parser.get_records(len(expected_particle))
        self.assertEqual(result, expected_particle)

        self.assertEqual(self.tel_exception_callback_value, None)
        in_file.close()

        log.debug('===== END TEST GET MANY =====')

    def test_invalid_metadata_records(self):
        """
        Read data from a file containing invalid metadata records as well
        as valid metadata records and sensor data records.
        Verify that the sensor data records can be read correctly
        and that invalid metadata records are detected.
        File 5 has 3 invalid metadata records.
        File 6 has 6 invalid metadata records.
        """
        log.debug('===== START TEST INVALID METADATA RECOVERED =====')
        in_file = self.open_file(FILE5)
        parser = self.create_rec_parser(in_file)

        for expected in EXPECTED_FILE5:

            # Generate expected particle
            expected_particle = DostaAbcdjmDclRecoveredInstrumentDataParticle(expected)

            # Get record and verify.
            result = parser.get_records(1)
            self.assertEqual(result, [expected_particle])

        self.assertEqual(self.rec_exceptions_detected, 3)
        in_file.close()

        log.debug('===== START TEST INVALID METADATA TELEMETERED =====')
        in_file = self.open_file(FILE6)
        parser = self.create_tel_parser(in_file)

        # Generate a list of expected result particles.
        expected_particle = []
        for expected in EXPECTED_FILE6:
            particle = DostaAbcdjmDclTelemeteredInstrumentDataParticle(expected)
            expected_particle.append(particle)

        # In a single read, get all particles for this file.
        result = parser.get_records(len(expected_particle))
        self.assertEqual(result, expected_particle)

        self.assertEqual(self.tel_exceptions_detected, 6)
        in_file.close()

        log.debug('===== END TEST INVALID METADATA =====')

    def test_invalid_sensor_data_records(self):
        """
        Read data from a file containing invalid sensor data records.
        Verify that no instrument particles are produced
        and the correct number of exceptions are detected.
        """
        log.debug('===== START TEST INVALID SENSOR DATA RECOVERED =====')
        in_file = self.open_file(FILE7)
        parser = self.create_rec_parser(in_file)

        expected_exceptions = 15

        # Try to get records and verify that none are returned.
        result = parser.get_records(1)
        self.assertEqual(result, [])
        self.assertEqual(self.rec_exceptions_detected, expected_exceptions)

        in_file.close()

        log.debug('===== START TEST INVALID SENSOR DATA TELEMETERED =====')
        in_file = self.open_file(FILE7)
        parser = self.create_tel_parser(in_file)

        # Try to get records and verify that none are returned.
        result = parser.get_records(1)
        self.assertEqual(result, [])
        self.assertEqual(self.tel_exceptions_detected, expected_exceptions)

        in_file.close()

        log.debug('===== END TEST INVALID SENSOR DATA =====')

    def test_mid_state_start(self):
        """
        Test starting a parser with a state in the middle of processing.
        """
        log.debug('===== START TEST MID-STATE START RECOVERED =====')

        in_file = self.open_file(FILE3)

        # Start at the beginning of the 21st record (of 42 total).
        initial_state = {
            DostaStateKey.POSITION: 2738
        }

        parser = self.create_rec_parser(in_file, new_state=initial_state)

        # Generate a list of expected result particles.
        expected_particle = []
        for expected in EXPECTED_FILE3[-22: ]:
            particle = DostaAbcdjmDclRecoveredInstrumentDataParticle(expected)
            expected_particle.append(particle)

        # In a single read, get all particles for this file.
        result = parser.get_records(len(expected_particle))
        self.assertEqual(result, expected_particle)

        self.assertEqual(self.rec_exception_callback_value, None)
        in_file.close()

        log.debug('===== START TEST MID-STATE START TELEMETERED =====')

        in_file = self.open_file(FILE2)

        # Start at the beginning of the 33rd record (of 44 total).
        initial_state = {
            DostaStateKey.POSITION: 4079
        }

        parser = self.create_tel_parser(in_file, new_state=initial_state)

        # Generate a list of expected result particles.
        expected_particle = []
        for expected in EXPECTED_FILE2[-12: ]:
            particle = DostaAbcdjmDclTelemeteredInstrumentDataParticle(expected)
            expected_particle.append(particle)

        # In a single read, get all particles for this file.
        result = parser.get_records(len(expected_particle))
        self.assertEqual(result, expected_particle)

        self.assertEqual(self.tel_exception_callback_value, None)
        in_file.close()

        log.debug('===== END TEST MID-STATE START =====')

    def test_no_sensor_data(self):
        """
        Read a file containing no sensor data records
        and verify that no particles are produced.
        """
        log.debug('===== START TEST NO SENSOR DATA RECOVERED =====')
        in_file = self.open_file(FILE0)
        parser = self.create_rec_parser(in_file)

        # Try to get a record and verify that none are produced.
        result = parser.get_records(1)
        self.assertEqual(result, [])

        self.assertEqual(self.rec_exception_callback_value, None)
        in_file.close()

        log.debug('===== START TEST NO SENSOR DATA TELEMETERED =====')
        in_file = self.open_file(FILE0)
        parser = self.create_tel_parser(in_file)

        # Try to get a record and verify that none are produced.
        result = parser.get_records(1)
        self.assertEqual(result, [])

        self.assertEqual(self.tel_exception_callback_value, None)
        in_file.close()

        log.debug('===== END TEST SENSOR DATA =====')

    def test_set_state(self):
        """
        This test verifies that the state can be changed after starting.
        Some particles are read and then the parser state is modified to
        skip ahead or back.
        """
        log.debug('===== START TEST SET STATE RECOVERED =====')

        in_file = self.open_file(FILE2)
        parser = self.create_rec_parser(in_file)

        # Read and verify 5 particles (of the 44).
        for expected in EXPECTED_FILE2[ : 5]:

            # Generate expected particle
            expected_particle = DostaAbcdjmDclRecoveredInstrumentDataParticle(expected)

            # Get record and verify.
            result = parser.get_records(1)
            self.assertEqual(result, [expected_particle])

        # Skip ahead in the file so that we get the last 10 particles.
        new_state = {
            DostaStateKey.POSITION: 4301
        }

        # Set the state.
        parser.set_state(new_state)

        # Read and verify the last 10 particles.
        for expected in EXPECTED_FILE2[-10: ]:

            # Generate expected particle
            expected_particle = DostaAbcdjmDclRecoveredInstrumentDataParticle(expected)

            # Get record and verify.
            result = parser.get_records(1)
            self.assertEqual(result, [expected_particle])

        log.debug('===== START TEST SET STATE TELEMETERED =====')

        in_file = self.open_file(FILE3)
        parser = self.create_tel_parser(in_file)

        # Read and verify 30 particles (of the 42).
        for expected in EXPECTED_FILE3[ : 30]:

            # Generate expected particle
            expected_particle = DostaAbcdjmDclTelemeteredInstrumentDataParticle(expected)

            # Get record and verify.
            result = parser.get_records(1)
            self.assertEqual(result, [expected_particle])

        # Skip back in the file so that we get the last 25 particles.
        new_state = {
            DostaStateKey.POSITION: 2414,
        }

        # Set the state.
        parser.set_state(new_state)

        # Read and verify the last 25 particles.
        for expected in EXPECTED_FILE3[-25: ]:

            # Generate expected particle
            expected_particle = DostaAbcdjmDclTelemeteredInstrumentDataParticle(expected)

            # Get record and verify.
            result = parser.get_records(1)
            self.assertEqual(result, [expected_particle])

        log.debug('===== END TEST SET STATE =====')

    def test_simple(self):
        """
        Read data from a file and pull out data particles
        one at a time. Verify that the results are those we expected.
        """
        log.debug('===== START TEST SIMPLE RECOVERED =====')
        in_file = self.open_file(FILE1)
        parser = self.create_rec_parser(in_file)

        for expected in EXPECTED_FILE1:

            # Generate expected particle
            expected_particle = DostaAbcdjmDclRecoveredInstrumentDataParticle(expected)

            # Get record and verify.
            result = parser.get_records(1)
            self.assertEqual(result, [expected_particle])

        self.assertEqual(self.rec_exception_callback_value, None)
        in_file.close()

        log.debug('===== START TEST SIMPLE TELEMETERED =====')
        in_file = self.open_file(FILE1)
        parser = self.create_tel_parser(in_file)

        for expected in EXPECTED_FILE1:

            # Generate expected particle
            expected_particle = DostaAbcdjmDclTelemeteredInstrumentDataParticle(expected)

            # Get record and verify.
            result = parser.get_records(1)
            self.assertEqual(result, [expected_particle])

        self.assertEqual(self.tel_exception_callback_value, None)
        in_file.close()

        log.debug('===== END TEST SIMPLE =====')

    def test_many_with_yml(self):
        """
        Read a file and verify that all records can be read.
        Verify that the contents of the particles are correct.
        There should be no exceptions generated.
        """
        log.debug('===== START TEST MANY WITH YML RECOVERED =====')

        num_particles = 21

        in_file = self.open_file(FILE1)
        parser = self.create_rec_parser(in_file)

        particles = parser.get_records(num_particles)

        log.debug("Num particles: %d", len(particles))

        self.assert_particles(particles, "rec_20010121.dosta1.yml", RESOURCE_PATH)
        self.assertEquals(self.exception_callback_value, [])

        in_file.close()

        log.debug('===== START TEST MANY WITH YML TELEMETERED =====')

        in_file = self.open_file(FILE1)
        parser = self.create_tel_parser(in_file)

        particles = parser.get_records(num_particles)

        log.debug("Num particles: %d", len(particles))

        self.assert_particles(particles, "tel_20010121.dosta1.yml", RESOURCE_PATH)
        self.assertEquals(self.exception_callback_value, [])

        in_file.close()

        log.debug('===== END TEST MANY WITH YML =====')
#!/usr/bin/env python

"""
@package mi.dataset.parser.test.test_dosta_abcdjm_dcl
@file marine-integrations/mi/dataset/parser/test/test_dosta_abcdjm_dcl.py
@author <NAME>
@brief Test code for a Dosta_abcdjm_dcl data parser

In the following files, Metadata consists of 4 records
and Garbled consist of 3 records.
There is 1 group of Sensor Data records for each set of metadata.

Files used for testing:

20000101.dosta0.log
  Metadata - 1 set,  Sensor Data - 0 records,  Garbled - 0,  Newline - \n

20010121.dosta1.log
  Metadata - 1 set,  Sensor Data - 21 records,  Garbled - 0,  Newline - \n

20020222.dosta2.log
  Metadata - 2 sets,  Sensor Data - 22 records,  Garbled - 0,  Newline - \r\n

20030314.dosta3.log
  Metadata - 3 sets,  Sensor Data - 14 records,  Garbled - 0,  Newline - \n

20041225.dosta4.log
  Metadata - 2 sets,  Sensor Data - 250 records,  Garbled - 0,  Newline - \n

20050103.dosta5.log
   Metadata - 1 set,  Sensor Data - 3 records,  Garbled - 1,  Newline - \n

20060207.dosta6.log
  Metadata - 2 sets,  Sensor Data - 7 records,  Garbled - 2,  Newline \r\n

20070114.dosta7.log
  This file contains a boatload of invalid sensor data records.  Newline - \r\n
   1. invalid year
   2. invalid month
   3. invalid day
   4. invalid hour
   5. invalid minute
   6. invalid second
   7. invalid product
   8. spaces instead of tabs
   9. a 2-digit serial number
  10. floating point number missing the decimal point
  11. serial number missing
  12. one of the floating point numbers missing
  13. Date in form YYYY-MM-DD
  14. time field missing milliseconds
  15. extra floating point number in sensor data
"""

import unittest
import os
from nose.plugins.attrib import attr

from mi.core.log import get_logger ; log = get_logger()

from mi.dataset.test.test_parser import ParserUnitTestCase

from mi.dataset.parser.dosta_abcdjm_dcl import \
    DostaAbcdjmDclParser, \
    DostaAbcdjmDclRecoveredParser, \
    DostaAbcdjmDclTelemeteredParser, \
    DostaAbcdjmDclRecoveredInstrumentDataParticle, \
    DostaAbcdjmDclTelemeteredInstrumentDataParticle, \
    DostaStateKey

from mi.dataset.dataset_parser import DataSetDriverConfigKeys

from mi.idk.config import Config
RESOURCE_PATH = os.path.join(Config().base_dir(), 'mi', 'dataset', 'driver',
                 'dosta_abcdjm', 'dcl', 'resource')


# Expected tuples for data in file 20010121.dosta1.log
# This file has no sensor data records and should not produce any particles.

# Expected tuples for data in file 20010121.dosta1.log
EXPECTED_20010121_dosta1 = [
    ('2001/01/21 03:06:02.329', '2001', '01', '21', '03', '06', '02', '329', '4831', '2001', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000'),
    ('2001/01/21 03:06:15.908', '2001', '01', '21', '03', '06', '15', '908', '4831', '2001', '25.000', '6.000', '5.000', '4.500', '4.500', '4.500', '4.500', '75.000', '75.000', '12.500'),
    ('2001/01/21 03:06:29.487', '2001', '01', '21', '03', '06', '29', '487', '4831', '2001', '50.000', '12.000', '10.000', '9.000', '9.000', '9.000', '9.000', '150.000', '150.000', '25.000'),
    ('2001/01/21 03:06:43.066', '2001', '01', '21', '03', '06', '43', '066', '4831', '2001', '75.000', '18.000', '15.000', '13.500', '13.500', '13.500', '13.500', '225.000', '225.000', '37.500'),
    ('2001/01/21 03:06:56.645', '2001', '01', '21', '03', '06', '56', '645', '4831', '2001', '100.000', '24.000', '20.000', '18.000', '18.000', '18.000', '18.000', '300.000', '300.000', '50.000'),
    ('2001/01/21 03:07:10.224', '2001', '01', '21', '03', '07', '10', '224', '4831', '2001', '125.000', '30.000', '25.000', '22.500', '22.500', '22.500', '22.500', '375.000', '375.000', '62.500'),
    ('2001/01/21 03:07:23.803', '2001', '01', '21', '03', '07', '23', '803', '4831', '2001', '150.000', '36.000', '30.000', '27.000', '27.000', '27.000', '27.000', '450.000', '450.000', '75.000'),
    ('2001/01/21 03:07:37.382', '2001', '01', '21', '03', '07', '37', '382', '4831', '2001', '175.000', '42.000', '35.000', '31.500', '31.500', '31.500', '31.500', '525.000', '525.000', '87.500'),
    ('2001/01/21 03:07:50.961', '2001', '01', '21', '03', '07', '50', '961', '4831', '2001', '200.000', '48.000', '40.000', '36.000', '36.000', '36.000', '36.000', '600.000', '600.000', '100.000'),
    ('2001/01/21 03:08:04.540', '2001', '01', '21', '03', '08', '04', '540', '4831', '2001', '225.000', '54.000', '45.000', '40.500', '40.500', '40.500', '40.500', '675.000', '675.000', '112.500'),
    ('2001/01/21 03:08:18.119', '2001', '01', '21', '03', '08', '18', '119', '4831', '2001', '250.000', '60.000', '50.000', '45.000', '45.000', '45.000', '45.000', '750.000', '750.000', '125.000'),
    ('2001/01/21 03:08:31.698', '2001', '01', '21', '03', '08', '31', '698', '4831', '2001', '275.000', '66.000', '55.000', '49.500', '49.500', '49.500', '49.500', '825.000', '825.000', '137.500'),
    ('2001/01/21 03:08:45.277', '2001', '01', '21', '03', '08', '45', '277', '4831', '2001', '300.000', '72.000', '60.000', '54.000', '54.000', '54.000', '54.000', '900.000', '900.000', '150.000'),
    ('2001/01/21 03:08:58.856', '2001', '01', '21', '03', '08', '58', '856', '4831', '2001', '325.000', '78.000', '65.000', '58.500', '58.500', '58.500', '58.500', '975.000', '975.000', '162.500'),
    ('2001/01/21 03:09:12.435', '2001', '01', '21', '03', '09', '12', '435', '4831', '2001', '350.000', '84.000', '70.000', '63.000', '63.000', '63.000', '63.000', '1050.000', '1050.000', '175.000'),
    ('2001/01/21 03:09:26.014', '2001', '01', '21', '03', '09', '26', '014', '4831', '2001', '375.000', '90.000', '75.000', '67.500', '67.500', '67.500', '67.500', '1125.000', '1125.000', '187.500'),
    ('2001/01/21 03:09:39.593', '2001', '01', '21', '03', '09', '39', '593', '4831', '2001', '400.000', '96.000', '80.000', '72.000', '72.000', '72.000', '72.000', '1200.000', '1200.000', '200.000'),
    ('2001/01/21 03:09:53.172', '2001', '01', '21', '03', '09', '53', '172', '4831', '2001', '425.000', '102.000', '85.000', '76.500', '76.500', '76.500', '76.500', '1275.000', '1275.000', '212.500'),
    ('2001/01/21 03:10:06.751', '2001', '01', '21', '03', '10', '06', '751', '4831', '2001', '450.000', '108.000', '90.000', '81.000', '81.000', '81.000', '81.000', '1350.000', '1350.000', '225.000'),
    ('2001/01/21 03:10:20.330', '2001', '01', '21', '03', '10', '20', '330', '4831', '2001', '475.000', '114.000', '95.000', '85.500', '85.500', '85.500', '85.500', '1425.000', '1425.000', '237.500'),
    ('2001/01/21 03:10:33.909', '2001', '01', '21', '03', '10', '33', '909', '4831', '2001', '500.000', '120.000', '100.000', '90.000', '90.000', '90.000', '90.000', '1500.000', '1500.000', '250.000'),
]

# Expected tuples for data in file 20020222.dosta2.log
EXPECTED_20020222_dosta2 = [
    ('2002/02/22 05:08:04.331', '2002', '02', '22', '05', '08', '04', '331', '4831', '2002', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000'),
    ('2002/02/22 05:08:17.910', '2002', '02', '22', '05', '08', '17', '910', '4831', '2002', '11.628', '2.791', '2.326', '2.093', '2.093', '2.093', '2.093', '34.884', '34.884', '5.814'),
    ('2002/02/22 05:08:31.489', '2002', '02', '22', '05', '08', '31', '489', '4831', '2002', '23.256', '5.581', '4.651', '4.186', '4.186', '4.186', '4.186', '69.767', '69.767', '11.628'),
    ('2002/02/22 05:08:45.068', '2002', '02', '22', '05', '08', '45', '068', '4831', '2002', '34.884', '8.372', '6.977', '6.279', '6.279', '6.279', '6.279', '104.651', '104.651', '17.442'),
    ('2002/02/22 05:08:58.647', '2002', '02', '22', '05', '08', '58', '647', '4831', '2002', '46.512', '11.163', '9.302', '8.372', '8.372', '8.372', '8.372', '139.535', '139.535', '23.256'),
    ('2002/02/22 05:09:12.226', '2002', '02', '22', '05', '09', '12', '226', '4831', '2002', '58.140', '13.953', '11.628', '10.465', '10.465', '10.465', '10.465', '174.419', '174.419', '29.070'),
    ('2002/02/22 05:09:25.805', '2002', '02', '22', '05', '09', '25', '805', '4831', '2002', '69.767', '16.744', '13.953', '12.558', '12.558', '12.558', '12.558', '209.302', '209.302', '34.884'),
    ('2002/02/22 05:09:39.384', '2002', '02', '22', '05', '09', '39', '384', '4831', '2002', '81.395', '19.535', '16.279', '14.651', '14.651', '14.651', '14.651', '244.186', '244.186', '40.698'),
    ('2002/02/22 05:09:52.963', '2002', '02', '22', '05', '09', '52', '963', '4831', '2002', '93.023', '22.326', '18.605', '16.744', '16.744', '16.744', '16.744', '279.070', '279.070', '46.512'),
    ('2002/02/22 05:10:06.542', '2002', '02', '22', '05', '10', '06', '542', '4831', '2002', '104.651', '25.116', '20.930', '18.837', '18.837', '18.837', '18.837', '313.953', '313.953', '52.326'),
    ('2002/02/22 05:10:20.121', '2002', '02', '22', '05', '10', '20', '121', '4831', '2002', '116.279', '27.907', '23.256', '20.930', '20.930', '20.930', '20.930', '348.837', '348.837', '58.140'),
    ('2002/02/22 05:10:33.700', '2002', '02', '22', '05', '10', '33', '700', '4831', '2002', '127.907', '30.698', '25.581', '23.023', '23.023', '23.023', '23.023', '383.721', '383.721', '63.953'),
    ('2002/02/22 05:10:47.279', '2002', '02', '22', '05', '10', '47', '279', '4831', '2002', '139.535', '33.488', '27.907', '25.116', '25.116', '25.116', '25.116', '418.605', '418.605', '69.767'),
    ('2002/02/22 05:11:00.858', '2002', '02', '22', '05', '11', '00', '858', '4831', '2002', '151.163', '36.279', '30.233', '27.209', '27.209', '27.209', '27.209', '453.488', '453.488', '75.581'),
    ('2002/02/22 05:11:14.437', '2002', '02', '22', '05', '11', '14', '437', '4831', '2002', '162.791', '39.070', '32.558', '29.302', '29.302', '29.302', '29.302', '488.372', '488.372', '81.395'),
    ('2002/02/22 05:11:28.016', '2002', '02', '22', '05', '11', '28', '016', '4831', '2002', '174.419', '41.860', '34.884', '31.395', '31.395', '31.395', '31.395', '523.256', '523.256', '87.209'),
    ('2002/02/22 05:11:41.595', '2002', '02', '22', '05', '11', '41', '595', '4831', '2002', '186.047', '44.651', '37.209', '33.488', '33.488', '33.488', '33.488', '558.140', '558.140', '93.023'),
    ('2002/02/22 05:11:55.174', '2002', '02', '22', '05', '11', '55', '174', '4831', '2002', '197.674', '47.442', '39.535', '35.581', '35.581', '35.581', '35.581', '593.023', '593.023', '98.837'),
    ('2002/02/22 05:12:08.753', '2002', '02', '22', '05', '12', '08', '753', '4831', '2002', '209.302', '50.233', '41.860', '37.674', '37.674', '37.674', '37.674', '627.907', '627.907', '104.651'),
    ('2002/02/22 05:12:22.332', '2002', '02', '22', '05', '12', '22', '332', '4831', '2002', '220.930', '53.023', '44.186', '39.767', '39.767', '39.767', '39.767', '662.791', '662.791', '110.465'),
    ('2002/02/22 05:12:35.911', '2002', '02', '22', '05', '12', '35', '911', '4831', '2002', '232.558', '55.814', '46.512', '41.860', '41.860', '41.860', '41.860', '697.674', '697.674', '116.279'),
    ('2002/02/22 05:12:49.490', '2002', '02', '22', '05', '12', '49', '490', '4831', '2002', '244.186', '58.605', '48.837', '43.953', '43.953', '43.953', '43.953', '732.558', '732.558', '122.093'),
    ('2002/02/22 05:13:57.385', '2002', '02', '22', '05', '13', '57', '385', '4831', '2002', '255.814', '61.395', '51.163', '46.047', '46.047', '46.047', '46.047', '767.442', '767.442', '127.907'),
    ('2002/02/22 05:14:10.964', '2002', '02', '22', '05', '14', '10', '964', '4831', '2002', '267.442', '64.186', '53.488', '48.140', '48.140', '48.140', '48.140', '802.326', '802.326', '133.721'),
    ('2002/02/22 05:14:24.543', '2002', '02', '22', '05', '14', '24', '543', '4831', '2002', '279.070', '66.977', '55.814', '50.233', '50.233', '50.233', '50.233', '837.209', '837.209', '139.535'),
    ('2002/02/22 05:14:38.122', '2002', '02', '22', '05', '14', '38', '122', '4831', '2002', '290.698', '69.767', '58.140', '52.326', '52.326', '52.326', '52.326', '872.093', '872.093', '145.349'),
    ('2002/02/22 05:14:51.701', '2002', '02', '22', '05', '14', '51', '701', '4831', '2002', '302.326', '72.558', '60.465', '54.419', '54.419', '54.419', '54.419', '906.977', '906.977', '151.163'),
    ('2002/02/22 05:15:05.280', '2002', '02', '22', '05', '15', '05', '280', '4831', '2002', '313.953', '75.349', '62.791', '56.512', '56.512', '56.512', '56.512', '941.860', '941.860', '156.977'),
    ('2002/02/22 05:15:18.859', '2002', '02', '22', '05', '15', '18', '859', '4831', '2002', '325.581', '78.140', '65.116', '58.605', '58.605', '58.605', '58.605', '976.744', '976.744', '162.791'),
    ('2002/02/22 05:15:32.438', '2002', '02', '22', '05', '15', '32', '438', '4831', '2002', '337.209', '80.930', '67.442', '60.698', '60.698', '60.698', '60.698', '1011.628', '1011.628', '168.605'),
    ('2002/02/22 05:15:46.017', '2002', '02', '22', '05', '15', '46', '017', '4831', '2002', '348.837', '83.721', '69.767', '62.791', '62.791', '62.791', '62.791', '1046.512', '1046.512', '174.419'),
    ('2002/02/22 05:15:59.596', '2002', '02', '22', '05', '15', '59', '596', '4831', '2002', '360.465', '86.512', '72.093', '64.884', '64.884', '64.884', '64.884', '1081.395', '1081.395', '180.233'),
    ('2002/02/22 05:16:13.175', '2002', '02', '22', '05', '16', '13', '175', '4831', '2002', '372.093', '89.302', '74.419', '66.977', '66.977', '66.977', '66.977', '1116.279', '1116.279', '186.047'),
    ('2002/02/22 05:16:26.754', '2002', '02', '22', '05', '16', '26', '754', '4831', '2002', '383.721', '92.093', '76.744', '69.070', '69.070', '69.070', '69.070', '1151.163', '1151.163', '191.860'),
    ('2002/02/22 05:16:40.333', '2002', '02', '22', '05', '16', '40', '333', '4831', '2002', '395.349', '94.884', '79.070', '71.163', '71.163', '71.163', '71.163', '1186.047', '1186.047', '197.674'),
    ('2002/02/22 05:16:53.912', '2002', '02', '22', '05', '16', '53', '912', '4831', '2002', '406.977', '97.674', '81.395', '73.256', '73.256', '73.256', '73.256', '1220.930', '1220.930', '203.488'),
    ('2002/02/22 05:17:07.491', '2002', '02', '22', '05', '17', '07', '491', '4831', '2002', '418.605', '100.465', '83.721', '75.349', '75.349', '75.349', '75.349', '1255.814', '1255.814', '209.302'),
    ('2002/02/22 05:17:21.070', '2002', '02', '22', '05', '17', '21', '070', '4831', '2002', '430.233', '103.256', '86.047', '77.442', '77.442', '77.442', '77.442', '1290.698', '1290.698', '215.116'),
    ('2002/02/22 05:17:34.649', '2002', '02', '22', '05', '17', '34', '649', '4831', '2002', '441.860', '106.047', '88.372', '79.535', '79.535', '79.535', '79.535', '1325.581', '1325.581', '220.930'),
    ('2002/02/22 05:17:48.228', '2002', '02', '22', '05', '17', '48', '228', '4831', '2002', '453.488', '108.837', '90.698', '81.628', '81.628', '81.628', '81.628', '1360.465', '1360.465', '226.744'),
    ('2002/02/22 05:18:01.807', '2002', '02', '22', '05', '18', '01', '807', '4831', '2002', '465.116', '111.628', '93.023', '83.721', '83.721', '83.721', '83.721', '1395.349', '1395.349', '232.558'),
    ('2002/02/22 05:18:15.386', '2002', '02', '22', '05', '18', '15', '386', '4831', '2002', '476.744', '114.419', '95.349', '85.814', '85.814', '85.814', '85.814', '1430.233', '1430.233', '238.372'),
    ('2002/02/22 05:18:28.965', '2002', '02', '22', '05', '18', '28', '965', '4831', '2002', '488.372', '117.209', '97.674', '87.907', '87.907', '87.907', '87.907', '1465.116', '1465.116', '244.186'),
    ('2002/02/22 05:18:42.544', '2002', '02', '22', '05', '18', '42', '544', '4831', '2002', '500.000', '120.000', '100.000', '90.000', '90.000', '90.000', '90.000', '1500.000', '1500.000', '250.000'),
]

# Expected tuples for data in file 20030314.dosta3.log
EXPECTED_20030314_dosta3 = [
    ('2003/03/14 07:10:06.333', '2003', '03', '14', '07', '10', '06', '333', '4831', '2003', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000'),
    ('2003/03/14 07:10:19.912', '2003', '03', '14', '07', '10', '19', '912', '4831', '2003', '12.195', '2.927', '2.439', '2.195', '2.195', '2.195', '2.195', '36.585', '36.585', '6.098'),
    ('2003/03/14 07:10:33.491', '2003', '03', '14', '07', '10', '33', '491', '4831', '2003', '24.390', '5.854', '4.878', '4.390', '4.390', '4.390', '4.390', '73.171', '73.171', '12.195'),
    ('2003/03/14 07:10:47.070', '2003', '03', '14', '07', '10', '47', '070', '4831', '2003', '36.585', '8.780', '7.317', '6.585', '6.585', '6.585', '6.585', '109.756', '109.756', '18.293'),
    ('2003/03/14 07:11:00.649', '2003', '03', '14', '07', '11', '00', '649', '4831', '2003', '48.780', '11.707', '9.756', '8.780', '8.780', '8.780', '8.780', '146.341', '146.341', '24.390'),
    ('2003/03/14 07:11:14.228', '2003', '03', '14', '07', '11', '14', '228', '4831', '2003', '60.976', '14.634', '12.195', '10.976', '10.976', '10.976', '10.976', '182.927', '182.927', '30.488'),
    ('2003/03/14 07:11:27.807', '2003', '03', '14', '07', '11', '27', '807', '4831', '2003', '73.171', '17.561', '14.634', '13.171', '13.171', '13.171', '13.171', '219.512', '219.512', '36.585'),
    ('2003/03/14 07:11:41.386', '2003', '03', '14', '07', '11', '41', '386', '4831', '2003', '85.366', '20.488', '17.073', '15.366', '15.366', '15.366', '15.366', '256.098', '256.098', '42.683'),
    ('2003/03/14 07:11:54.965', '2003', '03', '14', '07', '11', '54', '965', '4831', '2003', '97.561', '23.415', '19.512', '17.561', '17.561', '17.561', '17.561', '292.683', '292.683', '48.780'),
    ('2003/03/14 07:12:08.544', '2003', '03', '14', '07', '12', '08', '544', '4831', '2003', '109.756', '26.341', '21.951', '19.756', '19.756', '19.756', '19.756', '329.268', '329.268', '54.878'),
    ('2003/03/14 07:12:22.123', '2003', '03', '14', '07', '12', '22', '123', '4831', '2003', '121.951', '29.268', '24.390', '21.951', '21.951', '21.951', '21.951', '365.854', '365.854', '60.976'),
    ('2003/03/14 07:12:35.702', '2003', '03', '14', '07', '12', '35', '702', '4831', '2003', '134.146', '32.195', '26.829', '24.146', '24.146', '24.146', '24.146', '402.439', '402.439', '67.073'),
    ('2003/03/14 07:12:49.281', '2003', '03', '14', '07', '12', '49', '281', '4831', '2003', '146.341', '35.122', '29.268', '26.341', '26.341', '26.341', '26.341', '439.024', '439.024', '73.171'),
    ('2003/03/14 07:13:02.860', '2003', '03', '14', '07', '13', '02', '860', '4831', '2003', '158.537', '38.049', '31.707', '28.537', '28.537', '28.537', '28.537', '475.610', '475.610', '79.268'),
    ('2003/03/14 07:14:10.755', '2003', '03', '14', '07', '14', '10', '755', '4831', '2003', '170.732', '40.976', '34.146', '30.732', '30.732', '30.732', '30.732', '512.195', '512.195', '85.366'),
    ('2003/03/14 07:14:24.334', '2003', '03', '14', '07', '14', '24', '334', '4831', '2003', '182.927', '43.902', '36.585', '32.927', '32.927', '32.927', '32.927', '548.780', '548.780', '91.463'),
    ('2003/03/14 07:14:37.913', '2003', '03', '14', '07', '14', '37', '913', '4831', '2003', '195.122', '46.829', '39.024', '35.122', '35.122', '35.122', '35.122', '585.366', '585.366', '97.561'),
    ('2003/03/14 07:14:51.492', '2003', '03', '14', '07', '14', '51', '492', '4831', '2003', '207.317', '49.756', '41.463', '37.317', '37.317', '37.317', '37.317', '621.951', '621.951', '103.659'),
    ('2003/03/14 07:15:05.071', '2003', '03', '14', '07', '15', '05', '071', '4831', '2003', '219.512', '52.683', '43.902', '39.512', '39.512', '39.512', '39.512', '658.537', '658.537', '109.756'),
    ('2003/03/14 07:15:18.650', '2003', '03', '14', '07', '15', '18', '650', '4831', '2003', '231.707', '55.610', '46.341', '41.707', '41.707', '41.707', '41.707', '695.122', '695.122', '115.854'),
    ('2003/03/14 07:15:32.229', '2003', '03', '14', '07', '15', '32', '229', '4831', '2003', '243.902', '58.537', '48.780', '43.902', '43.902', '43.902', '43.902', '731.707', '731.707', '121.951'),
    ('2003/03/14 07:15:45.808', '2003', '03', '14', '07', '15', '45', '808', '4831', '2003', '256.098', '61.463', '51.220', '46.098', '46.098', '46.098', '46.098', '768.293', '768.293', '128.049'),
    ('2003/03/14 07:15:59.387', '2003', '03', '14', '07', '15', '59', '387', '4831', '2003', '268.293', '64.390', '53.659', '48.293', '48.293', '48.293', '48.293', '804.878', '804.878', '134.146'),
    ('2003/03/14 07:16:12.966', '2003', '03', '14', '07', '16', '12', '966', '4831', '2003', '280.488', '67.317', '56.098', '50.488', '50.488', '50.488', '50.488', '841.463', '841.463', '140.244'),
    ('2003/03/14 07:16:26.545', '2003', '03', '14', '07', '16', '26', '545', '4831', '2003', '292.683', '70.244', '58.537', '52.683', '52.683', '52.683', '52.683', '878.049', '878.049', '146.341'),
    ('2003/03/14 07:16:40.124', '2003', '03', '14', '07', '16', '40', '124', '4831', '2003', '304.878', '73.171', '60.976', '54.878', '54.878', '54.878', '54.878', '914.634', '914.634', '152.439'),
    ('2003/03/14 07:16:53.703', '2003', '03', '14', '07', '16', '53', '703', '4831', '2003', '317.073', '76.098', '63.415', '57.073', '57.073', '57.073', '57.073', '951.220', '951.220', '158.537'),
    ('2003/03/14 07:17:07.282', '2003', '03', '14', '07', '17', '07', '282', '4831', '2003', '329.268', '79.024', '65.854', '59.268', '59.268', '59.268', '59.268', '987.805', '987.805', '164.634'),
    ('2003/03/14 07:18:15.177', '2003', '03', '14', '07', '18', '15', '177', '4831', '2003', '341.463', '81.951', '68.293', '61.463', '61.463', '61.463', '61.463', '1024.390', '1024.390', '170.732'),
    ('2003/03/14 07:18:28.756', '2003', '03', '14', '07', '18', '28', '756', '4831', '2003', '353.659', '84.878', '70.732', '63.659', '63.659', '63.659', '63.659', '1060.976', '1060.976', '176.829'),
    ('2003/03/14 07:18:42.335', '2003', '03', '14', '07', '18', '42', '335', '4831', '2003', '365.854', '87.805', '73.171', '65.854', '65.854', '65.854', '65.854', '1097.561', '1097.561', '182.927'),
    ('2003/03/14 07:18:55.914', '2003', '03', '14', '07', '18', '55', '914', '4831', '2003', '378.049', '90.732', '75.610', '68.049', '68.049', '68.049', '68.049', '1134.146', '1134.146', '189.024'),
    ('2003/03/14 07:19:09.493', '2003', '03', '14', '07', '19', '09', '493', '4831', '2003', '390.244', '93.659', '78.049', '70.244', '70.244', '70.244', '70.244', '1170.732', '1170.732', '195.122'),
    ('2003/03/14 07:19:23.072', '2003', '03', '14', '07', '19', '23', '072', '4831', '2003', '402.439', '96.585', '80.488', '72.439', '72.439', '72.439', '72.439', '1207.317', '1207.317', '201.220'),
    ('2003/03/14 07:19:36.651', '2003', '03', '14', '07', '19', '36', '651', '4831', '2003', '414.634', '99.512', '82.927', '74.634', '74.634', '74.634', '74.634', '1243.902', '1243.902', '207.317'),
    ('2003/03/14 07:19:50.230', '2003', '03', '14', '07', '19', '50', '230', '4831', '2003', '426.829', '102.439', '85.366', '76.829', '76.829', '76.829', '76.829', '1280.488', '1280.488', '213.415'),
    ('2003/03/14 07:20:03.809', '2003', '03', '14', '07', '20', '03', '809', '4831', '2003', '439.024', '105.366', '87.805', '79.024', '79.024', '79.024', '79.024', '1317.073', '1317.073', '219.512'),
    ('2003/03/14 07:20:17.388', '2003', '03', '14', '07', '20', '17', '388', '4831', '2003', '451.220', '108.293', '90.244', '81.220', '81.220', '81.220', '81.220', '1353.659', '1353.659', '225.610'),
    ('2003/03/14 07:20:30.967', '2003', '03', '14', '07', '20', '30', '967', '4831', '2003', '463.415', '111.220', '92.683', '83.415', '83.415', '83.415', '83.415', '1390.244', '1390.244', '231.707'),
    ('2003/03/14 07:20:44.546', '2003', '03', '14', '07', '20', '44', '546', '4831', '2003', '475.610', '114.146', '95.122', '85.610', '85.610', '85.610', '85.610', '1426.829', '1426.829', '237.805'),
    ('2003/03/14 07:20:58.125', '2003', '03', '14', '07', '20', '58', '125', '4831', '2003', '487.805', '117.073', '97.561', '87.805', '87.805', '87.805', '87.805', '1463.415', '1463.415', '243.902'),
    ('2003/03/14 07:21:11.704', '2003', '03', '14', '07', '21', '11', '704', '4831', '2003', '500.000', '120.000', '100.000', '90.000', '90.000', '90.000', '90.000', '1500.000', '1500.000', '250.000'),
]

# Expected tuples for data in file 20041225.dosta4.log
# This file produces 500 particles.  It will be used for the large import test only.
# No verification of particles from this file are done in unit test.
# Particle results will be verified in integration and qualification tests.

# Expected tuples for data in file 20050103.dosta5.log
EXPECTED_20050103_dosta5 = [
    ('2005/01/03 11:14:10.337', '2005', '01', '03', '11', '14', '10', '337', '4831', '205', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000'),
    ('2005/01/03 11:14:23.916', '2005', '01', '03', '11', '14', '23', '916', '4831', '205', '250.000', '60.000', '50.000', '45.000', '45.000', '45.000', '45.000', '750.000', '750.000', '125.000'),
    ('2005/01/03 11:14:37.495', '2005', '01', '03', '11', '14', '37', '495', '4831', '205', '500.000', '120.000', '100.000', '90.000', '90.000', '90.000', '90.000', '1500.000', '1500.000', '250.000'),
]

# Expected tuples for data in file 20060207.dosta6.log
EXPECTED_20060207_dosta6 = [
    ('2006/02/07 13:16:12.339', '2006', '02', '07', '13', '16', '12', '339', '4831', '2006', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000', '0.000'),
    ('2006/02/07 13:16:25.918', '2006', '02', '07', '13', '16', '25', '918', '4831', '2006', '38.462', '9.231', '7.692', '6.923', '6.923', '6.923', '6.923', '115.385', '115.385', '19.231'),
    ('2006/02/07 13:16:39.497', '2006', '02', '07', '13', '16', '39', '497', '4831', '2006', '76.923', '18.462', '15.385', '13.846', '13.846', '13.846', '13.846', '230.769', '230.769', '38.462'),
    ('2006/02/07 13:16:53.076', '2006', '02', '07', '13', '16', '53', '076', '4831', '2006', '115.385', '27.692', '23.077', '20.769', '20.769', '20.769', '20.769', '346.154', '346.154', '57.692'),
    ('2006/02/07 13:17:06.655', '2006', '02', '07', '13', '17', '06', '655', '4831', '2006', '153.846', '36.923', '30.769', '27.692', '27.692', '27.692', '27.692', '461.538', '461.538', '76.923'),
    ('2006/02/07 13:17:20.234', '2006', '02', '07', '13', '17', '20', '234', '4831', '2006', '192.308', '46.154', '38.462', '34.615', '34.615', '34.615', '34.615', '576.923', '576.923', '96.154'),
    ('2006/02/07 13:17:33.813', '2006', '02', '07', '13', '17', '33', '813', '4831', '2006', '230.769', '55.385', '46.154', '41.538', '41.538', '41.538', '41.538', '692.308', '692.308', '115.385'),
    ('2006/02/07 13:19:22.445', '2006', '02', '07', '13', '19', '22', '445', '4831', '2006', '269.231', '64.615', '53.846', '48.462', '48.462', '48.462', '48.462', '807.692', '807.692', '134.615'),
    ('2006/02/07 13:19:36.024', '2006', '02', '07', '13', '19', '36', '024', '4831', '2006', '307.692', '73.846', '61.538', '55.385', '55.385', '55.385', '55.385', '923.077', '923.077', '153.846'),
    ('2006/02/07 13:19:49.603', '2006', '02', '07', '13', '19', '49', '603', '4831', '2006', '346.154', '83.077', '69.231', '62.308', '62.308', '62.308', '62.308', '1038.462', '1038.462', '173.077'),
    ('2006/02/07 13:20:03.182', '2006', '02', '07', '13', '20', '03', '182', '4831', '2006', '384.615', '92.308', '76.923', '69.231', '69.231', '69.231', '69.231', '1153.846', '1153.846', '192.308'),
    ('2006/02/07 13:20:16.761', '2006', '02', '07', '13', '20', '16', '761', '4831', '2006', '423.077', '101.538', '84.615', '76.154', '76.154', '76.154', '76.154', '1269.231', '1269.231', '211.538'),
    ('2006/02/07 13:20:30.340', '2006', '02', '07', '13', '20', '30', '340', '4831', '2006', '461.538', '110.769', '92.308', '83.077', '83.077', '83.077', '83.077', '1384.615', '1384.615', '230.769'),
    ('2006/02/07 13:20:43.919', '2006', '02', '07', '13', '20', '43', '919', '4831', '2006', '500.000', '120.000', '100.000', '90.000', '90.000', '90.000', '90.000', '1500.000', '1500.000', '250.000'),
]

FILE0 = '20000101.dosta0.log'
FILE1 = '20010121.dosta1.log'
FILE2 = '20020222.dosta2.log'
FILE3 = '20030314.dosta3.log'
FILE4 = '20041225.dosta4.log'
FILE5 = '20050103.dosta5.log'
FILE6 = '20060207.dosta6.log'
FILE7 = '20070114.dosta7.log'

EXPECTED_FILE1 = EXPECTED_20010121_dosta1
EXPECTED_FILE2 = EXPECTED_20020222_dosta2
EXPECTED_FILE3 = EXPECTED_20030314_dosta3
EXPECTED_FILE5 = EXPECTED_20050103_dosta5
EXPECTED_FILE6 = EXPECTED_20060207_dosta6


@attr('UNIT', group='mi')
class DostaAbcdjmDclParserUnitTestCase(ParserUnitTestCase):

    def create_rec_parser(self, file_handle, new_state=None):
        """
        This function creates a DostaAbcdjmDcl parser for recovered data.
        """
        parser = DostaAbcdjmDclRecoveredParser(self.rec_config,
            file_handle, new_state, self.rec_state_callback,
            self.rec_pub_callback, self.rec_exception_callback)
        return parser

    def create_tel_parser(self, file_handle, new_state=None):
        """
        This function creates a DostaAbcdjmDcl parser for telemetered data.
        """
        parser = DostaAbcdjmDclTelemeteredParser(self.tel_config,
            file_handle, new_state, self.rec_state_callback,
            self.tel_pub_callback, self.tel_exception_callback)
        return parser

    def open_file(self, filename):
        file = open(os.path.join(RESOURCE_PATH, filename), mode='r')
        return file

    def rec_state_callback(self, state, file_ingested):
        """ Call back method to watch what comes in via the position callback """
        self.rec_state_callback_value = state
        self.rec_file_ingested_value = file_ingested

    def tel_state_callback(self, state, file_ingested):
        """ Call back method to watch what comes in via the position callback """
        self.tel_state_callback_value = state
        self.tel_file_ingested_value = file_ingested

    def rec_pub_callback(self, pub):
        """ Call back method to watch what comes in via the publish callback """
        self.rec_publish_callback_value = pub

    def tel_pub_callback(self, pub):
        """ Call back method to watch what comes in via the publish callback """
        self.tel_publish_callback_value = pub

    def rec_exception_callback(self, exception):
        """ Call back method to watch what comes in via the exception callback """
        self.rec_exception_callback_value = exception
        self.rec_exceptions_detected += 1

    def tel_exception_callback(self, exception):
        """ Call back method to watch what comes in via the exception callback """
        self.tel_exception_callback_value = exception
        self.tel_exceptions_detected += 1

    def setUp(self):
        ParserUnitTestCase.setUp(self)

        self.rec_config = {
            DataSetDriverConfigKeys.PARTICLE_MODULE:
                'mi.dataset.parser.dosta_abcdjm_dcl',
            DataSetDriverConfigKeys.PARTICLE_CLASS:
                None
        }

        self.tel_config = {
            DataSetDriverConfigKeys.PARTICLE_MODULE:
                'mi.dataset.parser.dosta_abcdjm_dcl',
            DataSetDriverConfigKeys.PARTICLE_CLASS:
                None
        }

        self.rec_state_callback_value = None
        self.rec_file_ingested_value = False
        self.rec_publish_callback_value = None
        self.rec_exception_callback_value = None
        self.rec_exceptions_detected = 0

        self.tel_state_callback_value = None
        self.tel_file_ingested_value = False
        self.tel_publish_callback_value = None
        self.tel_exception_callback_value = None
        self.tel_exceptions_detected = 0

        self.maxDiff = None

    def test_big_giant_input(self):
        """
        Read a large file and verify that all expected particles can be read.
        Verification is not done at this time, but will be done during
        integration and qualification testing.
        File used for this test has 500 total particles.
        """
        log.debug('===== START TEST BIG GIANT INPUT RECOVERED =====')
        in_file = self.open_file(FILE4)
        parser = self.create_rec_parser(in_file)

        number_expected_results = 500

        # In a single read, get all particles in this file.
        result = parser.get_records(number_expected_results)
        self.assertEqual(len(result), number_expected_results)

        in_file.close()
        self.assertEqual(self.rec_exception_callback_value, None)

        log.debug('===== START TEST BIG GIANT INPUT TELEMETERED =====')
        in_file = self.open_file(FILE4)
        parser = self.create_tel_parser(in_file)

        # In a single read, get all particles in this file.
        result = parser.get_records(number_expected_results)
        self.assertEqual(len(result), number_expected_results)

        in_file.close()
        self.assertEqual(self.tel_exception_callback_value, None)

        log.debug('===== END TEST BIG GIANT INPUT =====')

    def test_get_many(self):
        """
        Read a file and pull out multiple data particles at one time.
        Verify that the results are those we expected.
        """
        log.debug('===== START TEST GET MANY RECOVERED =====')
        in_file = self.open_file(FILE3)
        parser = self.create_rec_parser(in_file)

        # Generate a list of expected result particles.
        expected_particle = []
        for expected in EXPECTED_FILE3:
            particle = DostaAbcdjmDclRecoveredInstrumentDataParticle(expected)
            expected_particle.append(particle)

        # In a single read, get all particles for this file.
        result = parser.get_records(len(expected_particle))
        self.assertEqual(result, expected_particle)

        self.assertEqual(self.rec_exception_callback_value, None)
        in_file.close()

        log.debug('===== START TEST GET MANY TELEMETERED =====')
        in_file = self.open_file(FILE3)
        parser = self.create_tel_parser(in_file)

        # Generate a list of expected result particles.
        expected_particle = []
        for expected in EXPECTED_FILE3:
            particle = DostaAbcdjmDclTelemeteredInstrumentDataParticle(expected)
            expected_particle.append(particle)

        # In a single read, get all particles for this file.
        result = parser.get_records(len(expected_particle))
        self.assertEqual(result, expected_particle)

        self.assertEqual(self.tel_exception_callback_value, None)
        in_file.close()

        log.debug('===== END TEST GET MANY =====')

    def test_invalid_metadata_records(self):
        """
        Read data from a file containing invalid metadata records as well
        as valid metadata records and sensor data records.
        Verify that the sensor data records can be read correctly
        and that invalid metadata records are detected.
        File 5 has 3 invalid metadata records.
        File 6 has 6 invalid metadata records.
        """
        log.debug('===== START TEST INVALID METADATA RECOVERED =====')
        in_file = self.open_file(FILE5)
        parser = self.create_rec_parser(in_file)

        for expected in EXPECTED_FILE5:

            # Generate expected particle
            expected_particle = DostaAbcdjmDclRecoveredInstrumentDataParticle(expected)

            # Get record and verify.
            result = parser.get_records(1)
            self.assertEqual(result, [expected_particle])

        self.assertEqual(self.rec_exceptions_detected, 3)
        in_file.close()

        log.debug('===== START TEST INVALID METADATA TELEMETERED =====')
        in_file = self.open_file(FILE6)
        parser = self.create_tel_parser(in_file)

        # Generate a list of expected result particles.
        expected_particle = []
        for expected in EXPECTED_FILE6:
            particle = DostaAbcdjmDclTelemeteredInstrumentDataParticle(expected)
            expected_particle.append(particle)

        # In a single read, get all particles for this file.
        result = parser.get_records(len(expected_particle))
        self.assertEqual(result, expected_particle)

        self.assertEqual(self.tel_exceptions_detected, 6)
        in_file.close()

        log.debug('===== END TEST INVALID METADATA =====')

    def test_invalid_sensor_data_records(self):
        """
        Read data from a file containing invalid sensor data records.
        Verify that no instrument particles are produced
        and the correct number of exceptions are detected.
        """
        log.debug('===== START TEST INVALID SENSOR DATA RECOVERED =====')
        in_file = self.open_file(FILE7)
        parser = self.create_rec_parser(in_file)

        expected_exceptions = 15

        # Try to get records and verify that none are returned.
        result = parser.get_records(1)
        self.assertEqual(result, [])
        self.assertEqual(self.rec_exceptions_detected, expected_exceptions)

        in_file.close()

        log.debug('===== START TEST INVALID SENSOR DATA TELEMETERED =====')
        in_file = self.open_file(FILE7)
        parser = self.create_tel_parser(in_file)

        # Try to get records and verify that none are returned.
        result = parser.get_records(1)
        self.assertEqual(result, [])
        self.assertEqual(self.tel_exceptions_detected, expected_exceptions)

        in_file.close()

        log.debug('===== END TEST INVALID SENSOR DATA =====')

    def test_mid_state_start(self):
        """
        Test starting a parser with a state in the middle of processing.
        """
        log.debug('===== START TEST MID-STATE START RECOVERED =====')

        in_file = self.open_file(FILE3)

        # Start at the beginning of the 21st record (of 42 total).
        initial_state = {
            DostaStateKey.POSITION: 2738
        }

        parser = self.create_rec_parser(in_file, new_state=initial_state)

        # Generate a list of expected result particles.
        expected_particle = []
        for expected in EXPECTED_FILE3[-22: ]:
            particle = DostaAbcdjmDclRecoveredInstrumentDataParticle(expected)
            expected_particle.append(particle)

        # In a single read, get all particles for this file.
        result = parser.get_records(len(expected_particle))
        self.assertEqual(result, expected_particle)

        self.assertEqual(self.rec_exception_callback_value, None)
        in_file.close()

        log.debug('===== START TEST MID-STATE START TELEMETERED =====')

        in_file = self.open_file(FILE2)

        # Start at the beginning of the 33rd record (of 44 total).
        initial_state = {
            DostaStateKey.POSITION: 4079
        }

        parser = self.create_tel_parser(in_file, new_state=initial_state)

        # Generate a list of expected result particles.
        expected_particle = []
        for expected in EXPECTED_FILE2[-12: ]:
            particle = DostaAbcdjmDclTelemeteredInstrumentDataParticle(expected)
            expected_particle.append(particle)

        # In a single read, get all particles for this file.
        result = parser.get_records(len(expected_particle))
        self.assertEqual(result, expected_particle)

        self.assertEqual(self.tel_exception_callback_value, None)
        in_file.close()

        log.debug('===== END TEST MID-STATE START =====')

    def test_no_sensor_data(self):
        """
        Read a file containing no sensor data records
        and verify that no particles are produced.
        """
        log.debug('===== START TEST NO SENSOR DATA RECOVERED =====')
        in_file = self.open_file(FILE0)
        parser = self.create_rec_parser(in_file)

        # Try to get a record and verify that none are produced.
        result = parser.get_records(1)
        self.assertEqual(result, [])

        self.assertEqual(self.rec_exception_callback_value, None)
        in_file.close()

        log.debug('===== START TEST NO SENSOR DATA TELEMETERED =====')
        in_file = self.open_file(FILE0)
        parser = self.create_tel_parser(in_file)

        # Try to get a record and verify that none are produced.
        result = parser.get_records(1)
        self.assertEqual(result, [])

        self.assertEqual(self.tel_exception_callback_value, None)
        in_file.close()

        log.debug('===== END TEST SENSOR DATA =====')

    def test_set_state(self):
        """
        This test verifies that the state can be changed after starting.
        Some particles are read and then the parser state is modified to
        skip ahead or back.
        """
        log.debug('===== START TEST SET STATE RECOVERED =====')

        in_file = self.open_file(FILE2)
        parser = self.create_rec_parser(in_file)

        # Read and verify 5 particles (of the 44).
        for expected in EXPECTED_FILE2[ : 5]:

            # Generate expected particle
            expected_particle = DostaAbcdjmDclRecoveredInstrumentDataParticle(expected)

            # Get record and verify.
            result = parser.get_records(1)
            self.assertEqual(result, [expected_particle])

        # Skip ahead in the file so that we get the last 10 particles.
        new_state = {
            DostaStateKey.POSITION: 4301
        }

        # Set the state.
        parser.set_state(new_state)

        # Read and verify the last 10 particles.
        for expected in EXPECTED_FILE2[-10: ]:

            # Generate expected particle
            expected_particle = DostaAbcdjmDclRecoveredInstrumentDataParticle(expected)

            # Get record and verify.
            result = parser.get_records(1)
            self.assertEqual(result, [expected_particle])

        log.debug('===== START TEST SET STATE TELEMETERED =====')

        in_file = self.open_file(FILE3)
        parser = self.create_tel_parser(in_file)

        # Read and verify 30 particles (of the 42).
        for expected in EXPECTED_FILE3[ : 30]:

            # Generate expected particle
            expected_particle = DostaAbcdjmDclTelemeteredInstrumentDataParticle(expected)

            # Get record and verify.
            result = parser.get_records(1)
            self.assertEqual(result, [expected_particle])

        # Skip back in the file so that we get the last 25 particles.
        new_state = {
            DostaStateKey.POSITION: 2414,
        }

        # Set the state.
        parser.set_state(new_state)

        # Read and verify the last 25 particles.
        for expected in EXPECTED_FILE3[-25: ]:

            # Generate expected particle
            expected_particle = DostaAbcdjmDclTelemeteredInstrumentDataParticle(expected)

            # Get record and verify.
            result = parser.get_records(1)
            self.assertEqual(result, [expected_particle])

        log.debug('===== END TEST SET STATE =====')

    def test_simple(self):
        """
        Read data from a file and pull out data particles
        one at a time. Verify that the results are those we expected.
        """
        log.debug('===== START TEST SIMPLE RECOVERED =====')
        in_file = self.open_file(FILE1)
        parser = self.create_rec_parser(in_file)

        for expected in EXPECTED_FILE1:

            # Generate expected particle
            expected_particle = DostaAbcdjmDclRecoveredInstrumentDataParticle(expected)

            # Get record and verify.
            result = parser.get_records(1)
            self.assertEqual(result, [expected_particle])

        self.assertEqual(self.rec_exception_callback_value, None)
        in_file.close()

        log.debug('===== START TEST SIMPLE TELEMETERED =====')
        in_file = self.open_file(FILE1)
        parser = self.create_tel_parser(in_file)

        for expected in EXPECTED_FILE1:

            # Generate expected particle
            expected_particle = DostaAbcdjmDclTelemeteredInstrumentDataParticle(expected)

            # Get record and verify.
            result = parser.get_records(1)
            self.assertEqual(result, [expected_particle])

        self.assertEqual(self.tel_exception_callback_value, None)
        in_file.close()

        log.debug('===== END TEST SIMPLE =====')

    def test_many_with_yml(self):
        """
        Read a file and verify that all records can be read.
        Verify that the contents of the particles are correct.
        There should be no exceptions generated.
        """
        log.debug('===== START TEST MANY WITH YML RECOVERED =====')

        num_particles = 21

        in_file = self.open_file(FILE1)
        parser = self.create_rec_parser(in_file)

        particles = parser.get_records(num_particles)

        log.debug("Num particles: %d", len(particles))

        self.assert_particles(particles, "rec_20010121.dosta1.yml", RESOURCE_PATH)
        self.assertEquals(self.exception_callback_value, [])

        in_file.close()

        log.debug('===== START TEST MANY WITH YML TELEMETERED =====')

        in_file = self.open_file(FILE1)
        parser = self.create_tel_parser(in_file)

        particles = parser.get_records(num_particles)

        log.debug("Num particles: %d", len(particles))

        self.assert_particles(particles, "tel_20010121.dosta1.yml", RESOURCE_PATH)
        self.assertEquals(self.exception_callback_value, [])

        in_file.close()

        log.debug('===== END TEST MANY WITH YML =====')
#!/usr/bin/env python @package mi.dataset.parser.test.test_dosta_abcdjm_dcl
 @file marine-integrations/mi/dataset/parser/test/test_dosta_abcdjm_dcl.py
 @author <NAME>
 @brief Test code for a Dosta_abcdjm_dcl data parser
 
 In the following files, Metadata consists of 4 records
 and Garbled consist of 3 records.
 There is 1 group of Sensor Data records for each set of metadata.
 
 Files used for testing:
 
 20000101.dosta0.log
   Metadata - 1 set,  Sensor Data - 0 records,  Garbled - 0,  Newline - \n
 
 20010121.dosta1.log
   Metadata - 1 set,  Sensor Data - 21 records,  Garbled - 0,  Newline - \n
 
 20020222.dosta2.log
   Metadata - 2 sets,  Sensor Data - 22 records,  Garbled - 0,  Newline - \r\n
 
 20030314.dosta3.log
   Metadata - 3 sets,  Sensor Data - 14 records,  Garbled - 0,  Newline - \n
 
 20041225.dosta4.log
   Metadata - 2 sets,  Sensor Data - 250 records,  Garbled - 0,  Newline - \n
 
 20050103.dosta5.log
    Metadata - 1 set,  Sensor Data - 3 records,  Garbled - 1,  Newline - \n
 
 20060207.dosta6.log
   Metadata - 2 sets,  Sensor Data - 7 records,  Garbled - 2,  Newline \r\n
 
 20070114.dosta7.log
   This file contains a boatload of invalid sensor data records.  Newline - \r\n
    1. invalid year
    2. invalid month
    3. invalid day
    4. invalid hour
    5. invalid minute
    6. invalid second
    7. invalid product
    8. spaces instead of tabs
    9. a 2-digit serial number
   10. floating point number missing the decimal point
   11. serial number missing
   12. one of the floating point numbers missing
   13. Date in form YYYY-MM-DD
   14. time field missing milliseconds
   15. extra floating point number in sensor data # Expected tuples for data in file 20010121.dosta1.log # This file has no sensor data records and should not produce any particles. # Expected tuples for data in file 20010121.dosta1.log # Expected tuples for data in file 20020222.dosta2.log # Expected tuples for data in file 20030314.dosta3.log # Expected tuples for data in file 20041225.dosta4.log # This file produces 500 particles.  It will be used for the large import test only. # No verification of particles from this file are done in unit test. # Particle results will be verified in integration and qualification tests. # Expected tuples for data in file 20050103.dosta5.log # Expected tuples for data in file 20060207.dosta6.log This function creates a DostaAbcdjmDcl parser for recovered data. This function creates a DostaAbcdjmDcl parser for telemetered data. Call back method to watch what comes in via the position callback Call back method to watch what comes in via the position callback Call back method to watch what comes in via the publish callback Call back method to watch what comes in via the publish callback Call back method to watch what comes in via the exception callback Call back method to watch what comes in via the exception callback Read a large file and verify that all expected particles can be read.
         Verification is not done at this time, but will be done during
         integration and qualification testing.
         File used for this test has 500 total particles. # In a single read, get all particles in this file. # In a single read, get all particles in this file. Read a file and pull out multiple data particles at one time.
         Verify that the results are those we expected. # Generate a list of expected result particles. # In a single read, get all particles for this file. # Generate a list of expected result particles. # In a single read, get all particles for this file. Read data from a file containing invalid metadata records as well
         as valid metadata records and sensor data records.
         Verify that the sensor data records can be read correctly
         and that invalid metadata records are detected.
         File 5 has 3 invalid metadata records.
         File 6 has 6 invalid metadata records. # Generate expected particle # Get record and verify. # Generate a list of expected result particles. # In a single read, get all particles for this file. Read data from a file containing invalid sensor data records.
         Verify that no instrument particles are produced
         and the correct number of exceptions are detected. # Try to get records and verify that none are returned. # Try to get records and verify that none are returned. Test starting a parser with a state in the middle of processing. # Start at the beginning of the 21st record (of 42 total). # Generate a list of expected result particles. # In a single read, get all particles for this file. # Start at the beginning of the 33rd record (of 44 total). # Generate a list of expected result particles. # In a single read, get all particles for this file. Read a file containing no sensor data records
         and verify that no particles are produced. # Try to get a record and verify that none are produced. # Try to get a record and verify that none are produced. This test verifies that the state can be changed after starting.
         Some particles are read and then the parser state is modified to
         skip ahead or back. # Read and verify 5 particles (of the 44). # Generate expected particle # Get record and verify. # Skip ahead in the file so that we get the last 10 particles. # Set the state. # Read and verify the last 10 particles. # Generate expected particle # Get record and verify. # Read and verify 30 particles (of the 42). # Generate expected particle # Get record and verify. # Skip back in the file so that we get the last 25 particles. # Set the state. # Read and verify the last 25 particles. # Generate expected particle # Get record and verify. Read data from a file and pull out data particles
         one at a time. Verify that the results are those we expected. # Generate expected particle # Get record and verify. # Generate expected particle # Get record and verify. Read a file and verify that all records can be read.
         Verify that the contents of the particles are correct.
         There should be no exceptions generated.