Auto-update archive list. #517

src/Archive/Redirect.coffee

@@ -1,5 +1,17 @@
 Redirect =
+  archives:
+    `<%=
+      JSON.stringify(readJSON('archives.json'), null, 2)
+        .replace(/\n {2,}(?!{)/g, ' ')
+        .replace(/\n/g, '\n    ')
+        .replace(/`/g, '\\`')
+    %>`
+
   init: ->
+    @selectArchives()
+    @update() if Conf['archiveAutoUpdate'] and Conf['lastarchivecheck'] < Date.now() - 2 * $.DAY
+
+  selectArchives: ->
     o =
       thread: {}
       post:   {}
@@ -7,8 +19,10 @@ Redirect =
       report: {}
 
     archives = {}
-    for data in Redirect.archives
+    for data in Conf['archives']
       {uid, name, boards, files, software, withCredentials} = data
+      boards = [] unless boards instanceof Array
+      files  = [] unless files  instanceof Array
       archives[JSON.stringify(uid ? name)] = data
       for boardID in boards
         unless withCredentials
@@ -27,13 +41,57 @@ Redirect =
 
     Redirect.data = o
 
-  archives:
-    `<%=
-      JSON.stringify(readJSON('archives.json'), null, 2)
-        .replace(/\n {2,}(?!{)/g, ' ')
-        .replace(/\n/g, '\n    ')
-        .replace(/`/g, '\\`')
-    %>`
+  update: (cb) ->
+    urls = []
+    responses = []
+    nloaded = 0
+    for url in Conf['archiveSources'].split('\n') when url[0] isnt '#'
+      url = url.trim()
+      urls.push url if url
+
+    load = (i) -> ->
+      fail = (action, msg) -> new Notice 'warning', "Error #{action} archive data from #{urls[i]}\n#{msg}", 20
+      return fail 'fetching', (if @status then "#{@status} #{@statusText}" else 'Connection Error') unless @status is 200
+      try
+        response = JSON.parse @response
+      catch err
+        return fail 'parsing', err.message
+      response = [response] unless response instanceof Array
+      responses[i] = response
+      nloaded++
+      if nloaded is urls.length
+        Redirect.parse responses, cb
+
+    if urls.length
+      for url, i in urls
+        if url[0] in ['[', '{']
+          load(i).call
+            status:   200
+            response: url
+        else
+          $.ajax url,
+            responseType: 'text'
+            onloadend: load(i)
+    else
+      Redirect.parse [], cb
+    return
+
+  parse: (responses, cb) ->
+    archives = []
+    archiveUIDs = {}
+    for response in responses
+      for data in response
+        uid = JSON.stringify(data.uid ? data.name)
+        if uid of archiveUIDs
+          $.extend archiveUIDs[uid], data
+        else
+          archiveUIDs[uid] = data
+          archives.push data
+    items = {archives, lastarchivecheck: Date.now()}
+    $.set items
+    $.extend Conf, items
+    Redirect.selectArchives()
+    cb?()
 
   to: (dest, data) ->
     archive = (if dest in ['search', 'board'] then Redirect.data.thread else Redirect.data[dest])[data.boardID]

src/General/Settings.coffee

@@ -430,21 +430,26 @@ Settings =
       inputs[input.name] = input
 
     items = {}
-    for name in ['captchaLanguage', 'boardnav', 'time', 'backlink', 'fileInfo', 'QR.personas', 'favicon', 'usercss', 'customCooldown']
+    for name in ['archiveSources', 'archiveAutoUpdate', 'captchaLanguage', 'boardnav', 'time', 'backlink', 'fileInfo', 'QR.personas', 'favicon', 'usercss', 'customCooldown']
       items[name] = Conf[name]
       input = inputs[name]
-      event = if name in ['QR.personas', 'favicon', 'usercss'] then 'change' else 'input'
-      $.on input, event, $.cb.value
+      event = if name in ['archiveSources', 'archiveAutoUpdate', 'QR.personas', 'favicon', 'usercss'] then 'change' else 'input'
+      $.on input, event, $.cb[if input.type is 'checkbox' then 'checked' else 'value']
       $.on input, event, Settings[name] if name of Settings
 
     $.get items, (items) ->
       for key, val of items
         input = inputs[key]
-        input.value = val
+        input[if input.type is 'checkbox' then 'checked' else 'value'] = val
         if key of Settings
           Settings[key].call input
       return
 
+    $.on inputs['archiveSources'], 'change', ->
+      $.set 'lastarchivecheck', 0
+      Conf['lastarchivecheck'] = 0
+      $.id('lastarchivecheck').textContent = 'never'
+
     interval  = inputs['Interval']
     customCSS = inputs['Custom CSS']
     applyCSS  = $ '#apply-css', section
@@ -458,8 +463,41 @@ Settings =
     $.on customCSS, 'change', Settings.togglecss
     $.on applyCSS,  'click',  -> CustomCSS.update()
 
+    itemsArchive = {}
+    itemsArchive[name] = Conf[name] for name in ['archives', 'selectedArchives', 'lastarchivecheck']
+    $.get itemsArchive, (itemsArchive) ->
+      $.extend Conf, itemsArchive
+      Settings.addArchiveTable section
+
+    boardSelect    = $ '#archive-board-select', section
+    table          = $ '#archive-table', section
+    updateArchives = $ '#update-archives', section
+
+    $.on boardSelect, 'change', ->
+      $('tbody > :not([hidden])', table).hidden = true
+      $("tbody > .#{@value}", table).hidden = false
+
+    $.on updateArchives, 'click', ->
+      Redirect.update ->
+        Settings.addArchiveTable section
+
+  addArchiveTable: (section) ->
+    $('#lastarchivecheck', section).textContent = if Conf['lastarchivecheck'] is 0
+      'never'
+    else
+      new Date(Conf['lastarchivecheck']).toLocaleString()
+
+    boardSelect = $ '#archive-board-select', section
+    table       = $ '#archive-table', section
+    tbody       = $ 'tbody', section
+
+    $.rmAll boardSelect
+    $.rmAll tbody
+
     archBoards = {}
-    for {uid, name, boards, files, software, withCredentials} in Redirect.archives
+    for {uid, name, boards, files, software, withCredentials} in Conf['archives']
+      boards = [] unless boards instanceof Array
+      files  = [] unless files  instanceof Array
       for boardID in boards
         o = archBoards[boardID] or=
           thread: [[], []]
@@ -492,24 +530,23 @@ Settings =
       $.add row, Settings.addArchiveCell boardID, o, item for item in ['thread', 'post', 'file']
       rows.push row
 
+    if rows.length is 0
+      boardSelect.hidden = table.hidden = true
+      return
+
+    boardSelect.hidden = table.hidden = false
+
     unless g.BOARD.ID of archBoards
       rows[0].hidden = false
 
-    $.add $('tbody', section), rows
-
-    boardSelect = $('#archive-board-select', section)
     $.add boardSelect, boardOptions
-    table = $('#archive-table', section)
-    $.on boardSelect, 'change', ->
-      $('tbody > :not([hidden])', table).hidden = true
-      $("tbody > .#{@value}", table).hidden = false
+    $.add tbody, rows
 
-    $.get 'selectedArchives', Conf['selectedArchives'], ({selectedArchives}) ->
-      for boardID, data of selectedArchives
-        for type, id of data
-          if select = $ "select[data-boardid='#{boardID}'][data-type='#{type}']", section
-            select.value = JSON.stringify id
-      return
+    for boardID, data of Conf['selectedArchives']
+      for type, id of data
+        if (select = $ "select[data-boardid='#{boardID}'][data-type='#{type}']", tbody)
+          select.value = JSON.stringify id
+          select.value = select.firstChild.value unless select.value
     return
 
   addArchiveCell: (boardID, data, type) ->
@@ -544,6 +581,7 @@ Settings =
     $.get 'selectedArchives', Conf['selectedArchives'], ({selectedArchives}) =>
       (selectedArchives[@dataset.boardid] or= {})[@dataset.type] = JSON.parse @value
       $.set 'selectedArchives', selectedArchives
+      Conf['selectedArchives'] = selectedArchives
 
   boardnav: ->
     Header.generateBoardList @value

src/General/Settings/Advanced.html

@@ -1,5 +1,5 @@
 <fieldset>
-  <legend>Archiver</legend>
+  <legend>Archives</legend>
   <div class="warning" data-feature="404 Redirect"><code>404 Redirect</code> is disabled.</div>
   <select id="archive-board-select"></select>
   <table id="archive-table">
@@ -10,6 +10,13 @@
     </thead>
     <tbody></tbody>
   </table>
+  <br>
+  <div>
+    Each line below should be an archive list in <a href="https://github.com/MayhemYDG/archives.json/blob/gh-pages/CONTRIBUTING.md" target="_blank">this format</a> or a URL to load an archive list from.<br>
+    Archive properties can be overriden by another item with the same <code>uid</code> (or if absent, its <code>name</code>).
+  </div>
+  <textarea name="archiveSources" class="field" spellcheck="false"></textarea>
+  <button id="update-archives">Update now</button> Last updated: <time id="lastarchivecheck"></time> <label><input type="checkbox" name="archiveAutoUpdate"> Auto-update</label>
 </fieldset>
 
 <fieldset>

src/config/Config.coffee

@@ -761,6 +761,11 @@ Config =
     'Shortcut Icons':             true
     'Custom Board Navigation':    true
 
+  archives:
+    archiveSources:    'https://4chan-x.just-believe.in/json/archives.json'
+    lastarchivecheck:  0
+    archiveAutoUpdate: true
+
   boardnav: """
     [ toggle-all ]
     a-replace

src/css/style.css

@@ -602,6 +602,9 @@ div[data-checked="false"] > .suboption-list {
 .section-advanced textarea {
   height: 150px;
 }
+.section-advanced textarea[name="archiveSources"] {
+  height: 75px;
+}
 .section-advanced .archive-cell {
   min-width: 160px;
   text-align: center;

src/main/Main.coffee

@@ -51,6 +51,7 @@ Main =
 
     for db in DataBoard.keys
       Conf[db] = boards: {}
+    Conf['archives'] = Redirect.archives
     Conf['selectedArchives'] = {}
     Conf['cooldowns'] = {}
     Conf['Index Sort'] = {}