/awips2/data/pg_log ERROR messages for duplicate data ingest

[mjames-upc]

aww

 grep "already exists" postgresql-Monday.log | cut -c 64-| grep aww |sed -e 's/2018-..-.._..:..:....//g'| sed -e 's/OSBM../OSBMXX/' | sed -e 's/OSBN../OSBNXX/' |sort | uniq
  Key (datauri)=(/aww//SEVERE WEATHER STATEMENT/KGLD//SVSGLD/0000/653 PM MDT SUN JUN 17 2018) already exists.
  Key (datauri)=(/aww//URGENT WEATHER MESSAGE/CYTR//WEATHE/0000/unknown) already exists.
  Key (datauri)=(/aww//URGENT WEATHER MESSAGE/KWBJ//OSBMXX/0000/unknown) already exists.
  Key (datauri)=(/aww//URGENT WEATHER MESSAGE/KWBJ//OSBNXX/0000/unknown) already exists.
  Key (datauri)=(/aww//URGENT WEATHER MESSAGE/KWNO//MMGNE1/0000/unknown) already exists.
  Key (datauri)=(/aww//URGENT WEATHER MESSAGE/KWNO//MMGSW1/0000/unknown) already exists.
  Key (datauri)=(/aww//URGENT WEATHER MESSAGE/RJTD//IMPORT/0000/unknown) already exists.
  Key (datauri)=(/aww//URGENT WEATHER MESSAGE/RJTD//VITAL /0000/unknown) already exists.
  Key (datauri)=(/aww//URGENT WEATHER MESSAGE/SABM///0000/unknown) already exists.
  Key (datauri)=(/aww//URGENT WEATHER MESSAGE/SBBR///0000/unknown) already exists.
  Key (datauri)=(/aww//WEATHER STATEMENT/KAPX//MWSAPX/0000/1057 PM EDT SUN JUN 17 2018) already exists.

bufrmosMRF

  Key (datauri)=(/bufrmosMRF/_(60)/PAMB/58.99/-159.05) already exists.
  Key (datauri)=(/bufrmosMRF/_(60)/PAJZ/59.730000000000004/-157.26) already exists.
  Key (datauri)=(/bufrmosMRF/_(66)/PALG/61.54/-160.34) already exists.
  Key (datauri)=(/bufrmosMRF/_(42)/PAVC/55.120000000000005/-162.27) already exists.
  Key (datauri)=(/bufrmosMRF/_(66)/PAIG/59.32/-155.9) already exists.
  Key (datauri)=(/bufrmosMRF/_(48)/PAVC/55.120000000000005/-162.27) already exists.
  Key (datauri)=(/bufrmosMRF/_(24)/PAMK/63.49/-162.11) already exists.
  Key (datauri)=(/bufrmosMRF/_(72)/PARY/64.73/-155.47) already exists.
  Key (datauri)=(/bufrmosMRF/_(60)/PACM/61.85/-165.57) already exists.
  Key (datauri)=(/bufrmosMRF/_(24)/PAGH/66.89/-157.16) already exists.
  Key (datauri)=(/bufrmosMRF/_(30)/PAMO/62.1/-163.68) already exists.
  Key (datauri)=(/bufrmosMRF/_(18)/PAMK/63.49/-162.11) already exists.
  Key (datauri)=(/bufrmosMRF/_(78)/PARY/64.73/-155.47) already exists.
  Key (datauri)=(/bufrmosMRF/_(30)/PFNO/66.82000000000001/-161.02) already exists.
  Key (datauri)=(/bufrmosMRF/_(102)/PAKF/54.85/-163.41) already exists.
  Key (datauri)=(/bufrmosMRF/_(126)/PAJZ/59.730000000000004/-157.26) already exists.
  Key (datauri)=(/bufrmosMRF/_(126)/PAVA/61.54/-165.6) already exists.
  Key (datauri)=(/bufrmosMRF/_(198)/PAVC/55.120000000000005/-162.27) already exists.
  Key (datauri)=(/bufrmosMRF/_(138)/PAKI/59.93/-164.03) already exists.
  Key (datauri)=(/bufrmosMRF/_(126)/PALG/61.54/-160.34) already exists.
  Key (datauri)=(/bufrmosMRF/_(174)/PANW/59.45/-157.33) already exists.
  Key (datauri)=(/bufrmosMRF/_(192)/PAOO/60.53/-165.11) already exists.
  Key (datauri)=(/bufrmosMRF/_(120)/PAMO/62.1/-163.68) already exists.
  Key (datauri)=(/bufrmosMRF/_(144)/PATQ/70.47/-157.44) already exists.
  Key (datauri)=(/bufrmosMRF/_(174)/PATQ/70.47/-157.44) already exists.
  Key (datauri)=(/bufrmosMRF/_(150)/PARY/64.73/-155.47) already exists.
  Key (datauri)=(/bufrmosMRF/_(180)/PARY/64.73/-155.47) already exists.
  Key (datauri)=(/bufrmosMRF/_(96)/PARY/64.73/-155.47) already exists.
  Key (datauri)=(/bufrmosMRF/_(96)/PAGH/66.89/-157.16) already exists.
  Key (datauri)=(/bufrmosMRF/_(96)/PARS/61.77/-161.32) already exists.
  Key (datauri)=(/bufrmosMRF/_(102)/PADM/61.86/-162.03) already exists.
  Key (datauri)=(/bufrmosMRF/_(114)/PAHC/62.190000000000005/-159.77) already exists.
  Key (datauri)=(/bufrmosMRF/_(180)/PAIK/66.98/-160.44) already exists.
  Key (datauri)=(/bufrmosMRF/_(138)/PAGH/66.89/-157.16) already exists.

ncpafm

 Key (datauri)=(/ncpafm/_(2)/KLOT//ILZ011/41.98/-88.69/FOUS53/POINT/PAFM) already exists.
  Key (datauri)=(/ncpafm/_(11)/KLOT//ILZ003/42.2/-89.11/FOUS53/POINT/PAFM) already exists.
  Key (datauri)=(/ncpafm/_(71)/KLOT//ILZ020/41.64/-88.45/FOUS53/POINT/PAFM) already exists.
  Key (datauri)=(/ncpafm/_(-7)/KLOT//ILZ012/41.88/-88.31/FOUS53/POINT/PAFM) already exists.
  Key (datauri)=(/ncpafm/_(53)/KLOT//ILZ039/40.45/-88.1/FOUS53/POINT/PAFM) already exists.
  Key (datauri)=(/ncpafm/_(149)/KLOT//ILZ023/41.12/-87.86/FOUS53/POINT/PAFM) already exists.
  Key (datauri)=(/ncpafm/_(149)/KLOT//ILZ039/40.45/-88.1/FOUS53/POINT/PAFM) already exists.
  Key (datauri)=(/ncpafm/_(100)/KLOT//INZ019/40.62/-87.32/FOUS53/POINT/PAFM) already exists.
  Key (datauri)=(/ncpafm/_(125)/KLOT//ILZ021/40.92/-88.63/FOUS53/POINT/PAFM) already exists.
  Key (datauri)=(/ncpafm/_(161)/KLOT//ILZ021/40.92/-88.63/FOUS53/POINT/PAFM) already exists.

unknown

  Key (latitude, longitude, stationid, reftime, forecasttime, eventtype)=(39.6199989, -104.040001, -104.04:39.62, 2018-06-17 22:02:00, 0, TSTM WND GST) already exists.
  Key (latitude, longitude, stationid, reftime, forecasttime, eventtype)=(39.6199989, -104.040001, -104.04:39.62, 2018-06-17 22:02:00, 0, TSTM WND GST) already exists.
  Key (latitude, longitude, stationid, reftime, forecasttime, eventtype)=(36.1100006, -86.8199997, -86.82:36.11, 2018-06-17 00:15:00, 0, TSTM WND DMG) already exists.

  Key (stationid, reftime, sattype, pressure, latitude, longitude)=(-144.285:26.848, 2018-06-17 22:24:00, WV7.4u, 43700, 26.8484993, -144.285202) already exists.
  Key (stationid, reftime, sattype, pressure, latitude, longitude)=(-144.285:26.848, 2018-06-17 22:24:00, WV7.4u, 43700, 26.8484993, -144.285202) already exists.
  Key (stationid, reftime, sattype, pressure, latitude, longitude)=(-144.285:26.848, 2018-06-17 22:24:00, WV7.4u, 43700, 26.8484993, -144.285202) already exists.
  Key (stationid, reftime, sattype, pressure, latitude, longitude)=(-153.318:24.345, 2018-06-17 22:24:00, WV7.4u, 26200, 24.3449993, -153.317902) already exists.
  Key (stationid, reftime, sattype, pressure, latitude, longitude)=(-143.715:27.494, 2018-06-17 22:24:00, WV7.4u, 43700, 27.4941998, -143.714706) already exists.
  Key (stationid, reftime, sattype, pressure, latitude, longitude)=(-146.402:27.753, 2018-06-17 22:24:00, WV7.4u, 42500, 27.7528992, -146.402206) already exists.
  Key (stationid, reftime, sattype, pressure, latitude, longitude)=(-157.976:47.14, 2018-06-17 22:24:00, WV7.4u, 37500, 47.1400986, -157.975906) already exists.
  Key (stationid, reftime, sattype, pressure, latitude, longitude)=(-150.873:25.244, 2018-06-17 22:24:00, WV7.4u, 28700, 25.2441006, -150.873093) already exists.
  Key (stationid, reftime, sattype, pressure, latitude, longitude)=(-147.294:27.989, 2018-06-17 22:24:00, WV7.4u, 36200, 27.9885006, -147.2939) already exists.
  Key (stationid, reftime, sattype, pressure, latitude, longitude)=(-152.609:26.464, 2018-06-17 22:24:00, WV7.4u, 27500, 26.4638004, -152.608704) already exists.
  Key (stationid, reftime, sattype, pressure, latitude, longitude)=(-140.983:24.049, 2018-06-17 22:24:00, WV7.4u, 46200, 24.0487003, -140.983307) already exists.
  Key (stationid, reftime, sattype, pressure, latitude, longitude)=(-148.191:26.694, 2018-06-17 22:24:00, WV7.4u, 45000, 26.6938992, -148.191498) already exists.

  Key (stationid, reftime, satid, latitude, longitude)=(-134.38:-32.16, 2018-06-17 23:43:58, 248, -32.1599998, -134.380005) already exists.
  Key (stationid, reftime, satid, latitude, longitude)=(-134.38:-32.16, 2018-06-17 23:43:58, 248, -32.1599998, -134.380005) already exists.
  Key (stationid, reftime, satid, latitude, longitude)=(-121.19:-30.04, 2018-06-17 23:43:58, 248, -30.0400009, -121.190002) already exists.
  Key (stationid, reftime, satid, latitude, longitude)=(-120.53:-30.41, 2018-06-17 23:43:58, 248, -30.4099998, -120.529999) already exists.
  Key (stationid, reftime, satid, latitude, longitude)=(-134.54:-32.34, 2018-06-17 23:43:58, 248, -32.3400002, -134.539993) already exists.
  Key (stationid, reftime, satid, latitude, longitude)=(-134.03:-31.81, 2018-06-17 23:43:58, 248, -31.8099995, -134.029999) already exists.
  Key (stationid, reftime, satid, latitude, longitude)=(-132.26:-30.44, 2018-06-17 23:43:58, 248, -30.4400005, -132.259995) already exists.
  Key (stationid, reftime, satid, latitude, longitude)=(-120.75:-30.28, 2018-06-17 23:43:58, 248, -30.2800007, -120.75) already exists.
  Key (stationid, reftime, satid, latitude, longitude)=(-131.37:-29.96, 2018-06-17 23:43:58, 248, -29.9599991, -131.369995) already exists.
  Key (stationid, reftime, satid, latitude, longitude)=(-130.44:-29.55, 2018-06-17 23:43:58, 248, -29.5499992, -130.440002) already exists.
  Key (stationid, reftime, satid, latitude, longitude)=(-122.84:-29.36, 2018-06-17 23:43:58, 248, -29.3600006, -122.839996) already exists.
  Key (stationid, reftime, satid, latitude, longitude)=(-125.11:-28.88, 2018-06-17 23:43:58, 248, -28.8799992, -125.110001) already exists.
  Key (stationid, reftime, satid, latitude, longitude)=(-129.21:-29.17, 2018-06-17 23:43:58, 248, -29.1700001, -129.210007) already exists.
  Key (stationid, reftime, satid, latitude, longitude)=(-124.6:-28.94, 2018-06-17 23:43:58, 248, -28.9400005, -124.599998) already exists.

 Key (cccid, datacrc, hdrtime, nnnid, site, wmoid, xxxid)=(RNO, 1944578462, 180213, STQ, KREV, BMBB91, REV) already exists.
 Key (cccid, datacrc, hdrtime, nnnid, site, wmoid, xxxid)=(SLC, 3545983916, 180257, STQ, KSLC, BMBB91, SLC) already exists.

  Key (reftime, reporttype, stationid, latitude, longitude)=(2018-06-17 19:54:00, A, DLH:4640:9216, 46.4000015, -92.1600037) already exists.
  Key (reftime, reporttype, stationid, latitude, longitude)=(2018-06-17 19:54:00, A, DLH:4640:9216, 46.4000015, -92.1600037) already exists.
  Key (reftime, reporttype, stationid, latitude, longitude)=(2018-06-17 20:45:00, W, OHX:3593:8684, 35.9300003, -86.8399963) already exists.
  Key (reftime, reporttype, stationid, latitude, longitude)=(2018-06-17 17:50:00, W, DLH:4673:9331, 46.7299995, -93.3099976) already exists.
  Key (reftime, reporttype, stationid, latitude, longitude)=(2018-06-17 20:35:00, W, OHX:3600:8679, 36, -86.7900009) already exists.
  Key (reftime, reporttype, stationid, latitude, longitude)=(2018-06-17 20:56:00, W, LBF:411110207, 41.1100006, -102.07) already exists.
  Key (reftime, reporttype, stationid, latitude, longitude)=(2018-06-17 21:00:00, W, GSP:3489:8246, 34.8899994, -82.4599991) already exists.
  Key (reftime, reporttype, stationid, latitude, longitude)=(2018-06-17 21:05:00, W, GSP:3488:8246, 34.8800011, -82.4599991) already exists.
  Key (reftime, reporttype, stationid, latitude, longitude)=(2018-06-17 19:50:00, W, OHX:3650:8688, 36.5, -86.8799973) already exists.
  Key (reftime, reporttype, stationid, latitude, longitude)=(2018-06-17 21:00:00, W, LBF:411110201, 41.1100006, -102.010002) already exists.
  Key (reftime, reporttype, stationid, latitude, longitude)=(2018-06-17 20:15:00, W, OHX:3600:8679, 36, -86.7900009) already exists.
  Key (reftime, reporttype, stationid, latitude, longitude)=(2018-06-17 20:20:00, T, BOU:410010229, 41, -102.290001) already exists.
  Key (reftime, reporttype, stationid, latitude, longitude)=(2018-06-17 21:25:00, W, BMX:3362:8709, 33.6199989, -87.0899963) already exists.
  Key (reftime, reporttype, stationid, latitude, longitude)=(2018-06-17 18:40:00, W, FFC:3447:8446, 34.4700012, -84.4599991) already exists.
  Key (reftime, reporttype, stationid, latitude, longitude)=(2018-06-17 20:28:00, T, LBF:410110223, 41.0099983, -102.230003) already exists.
  Key (reftime, reporttype, stationid, latitude, longitude)=(2018-06-17 18:53:00, W, GSP:3585:8274, 35.8499985, -82.7399979) already exists.
  Key (reftime, reporttype, stationid, latitude, longitude)=(2018-06-17 20:48:00, T, LBF:411610207, 41.1599998, -102.07) already exists.
  Key (reftime, reporttype, stationid, latitude, longitude)=(2018-06-17 17:54:00, W, PBZ:4022:8190, 40.2200012, -81.9000015) already exists.
  Key (reftime, reporttype, stationid, latitude, longitude)=(2018-06-17 20:07:00, W, PBZ:3981:8157, 39.8100014, -81.5699997) already exists.
  Key (reftime, reporttype, stationid, latitude, longitude)=(2018-06-17 20:31:00, A, OHX:3593:8684, 35.9300003, -86.8399963) already exists.
  Key (reftime, reporttype, stationid, latitude, longitude)=(2018-06-17 17:38:00, W, MPX:4623:9464, 46.2299995, -94.6399994) already exists.
  Key (reftime, reporttype, stationid, latitude, longitude)=(2018-06-17 20:35:00, W, LBF:410610236, 41.0600014, -102.360001) already exists.
  Key (reftime, reporttype, stationid, latitude, longitude)=(2018-06-17 19:00:00, W, FFC:3464:8453, 34.6399994, -84.5299988) already exists.
  Key (reftime, reporttype, stationid, latitude, longitude)=(2018-06-17 19:57:00, A, BOU:409710248, 40.9700012, -102.480003) already exists.
  Key (reftime, reporttype, stationid, latitude, longitude)=(2018-06-17 19:06:00, W, PBZ:3995:8201, 39.9500008, -82.0100021) already exists.
  Key (reftime, reporttype, stationid, latitude, longitude)=(2018-06-17 21:25:00, W, LBF:412410166, 41.2400017, -101.660004) already exists.
  Key (reftime, reporttype, stationid, latitude, longitude)=(2018-06-17 17:30:00, A, DLH:4668:9357, 46.6800003, -93.5699997) already exists.
  Key (reftime, reporttype, stationid, latitude, longitude)=(2018-06-17 21:05:00, W, GSP:3489:8245, 34.8899994, -82.4499969) already exists.
  Key (reftime, reporttype, stationid, latitude, longitude)=(2018-06-17 21:45:00, W, MPX:4550:9100, 45.5, -91) already exists.
  Key (reftime, reporttype, stationid, latitude, longitude)=(2018-06-17 18:55:00, W, FFC:3454:8453, 34.5400009, -84.5299988) already exists.
  Key (reftime, reporttype, stationid, latitude, longitude)=(2018-06-17 20:20:00, W, PBZ:3974:8159, 39.7400017, -81.5899963) already exists.
  Key (reftime, reporttype, stationid, latitude, longitude)=(2018-06-17 19:10:00, W, DLH:4654:9315, 46.5400009, -93.1500015) already exists.
  Key (reftime, reporttype, stationid, latitude, longitude)=(2018-06-17 20:54:00, W, LBF:410610207, 41.0600014, -102.07) already exists.
  Key (reftime, reporttype, stationid, latitude, longitude)=(2018-06-17 21:25:00, A, LBF:412410166, 41.2400017, -101.660004) already exists.
  Key (reftime, reporttype, stationid, latitude, longitude)=(2018-06-17 19:26:00, A, PIH:439211168, 43.9199982, -111.68) already exists.
  Key (reftime, reporttype, stationid, latitude, longitude)=(2018-06-17 21:28:00, W, OHX:3638:8645, 36.3800011, -86.4499969) already exists.
  Key (reftime, reporttype, stationid, latitude, longitude)=(2018-06-17 20:41:00, T, LBF:410910203, 41.0900002, -102.029999) already exists.
  Key (reftime, reporttype, stationid, latitude, longitude)=(2018-06-17 21:25:00, W, LBF:412510168, 41.25, -101.68) already exists.
  Key (reftime, reporttype, stationid, latitude, longitude)=(2018-06-17 21:01:00, W, MPX:4487:9149, 44.8699989, -91.4899979) already exists.

ncuair

grep "already exists" postgresql-Monday.log | cut -c 64-| grep -v aww | sed -e 's/2018-..-.._..:..:....//g' |grep ncuair
  Key (datauri)=(/ncuair//UAIR/03953/51.93/-10.25/TTXX/xx/null) already exists.
  Key (datauri)=(/ncuair//UAIR/20744/72.37/52.73/TTXX/xx/null) already exists.
  Key (datauri)=(/ncuair//UAIR/22271/67.88/44.13/TTXX/12/RRA) already exists.
  Key (datauri)=(/ncuair//UAIR/24125/68.5/112.43/TTXX/xx/null) already exists.
  Key (datauri)=(/ncuair//UAIR/26298/57.9/34.05/TTXX/xx/null) already exists.
  Key (datauri)=(/ncuair//UAIR/26702/54.7/20.62/TTXX/xx/null) already exists.
  Key (datauri)=(/ncuair//UAIR/27730/54.63/39.7/TTXX/xx/null) already exists.
  Key (datauri)=(/ncuair//UAIR/34009/51.77/36.17/TTXX/xx/null) already exists.
  Key (datauri)=(/ncuair//UAIR/34858/45.92/43.35/TTXX/xx/null) already exists.
  Key (datauri)=(/ncuair//UAIR/42874/21.23/81.65/TTXX/xx/null) already exists.
  Key (datauri)=(/ncuair//UAIR/43041/19.08/82.03/TTXX/xx/null) already exists.
  Key (datauri)=(/ncuair//UAIR/43192/15.48/73.82/TTXX/xx/null) already exists.
...

nucaps

grep "already exists" postgresql-Monday.log | cut -c 64-| grep nucaps | sed -e 's/2018-..-.._..:..:....//g' |more
  Key (datauri)=(/nucaps/[--]/34.80570983886719/-35.18421936035156) already exists.
  Key (datauri)=(/nucaps/[--]/37.46162414550781/-50.25530242919922) already exists.
  Key (datauri)=(/nucaps/[--]/-11.902082443237305/-90.51399230957031) already exists.
  Key (datauri)=(/nucaps/[--]/-12.679092407226562/-82.5501480102539) already exists.
  Key (datauri)=(/nucaps/[--]/-13.967201232910156/-86.9668960571289) already exists.
  Key (datauri)=(/nucaps/[--]/-16.366592407226562/-80.1377944946289) already exists.
  Key (datauri)=(/nucaps/[--]/-17.364011764526367/-83.69194030761719) already exists.
  Key (datauri)=(/nucaps/[--]/-16.03451156616211/-91.69319152832031) already exists.
  Key (datauri)=(/nucaps/[--]/-23.404510498046875/-81.7780532836914) already exists.
  Key (datauri)=(/nucaps/[--]/-23.369199752807617/-85.76189422607422) already exists.
  Key (datauri)=(/nucaps/[--]/-19.01141929626465/-82.18749237060547) already exists.
...