fetching custom metadata from op_* metadata instead of fetching them on metadata.json file

lib/podcast_feed/provider/archive/parser.ex

@@ -7,7 +7,6 @@ defmodule PodcastFeed.Provider.Archive.Parser do
   alias PodcastFeed.Utility.Format
   alias __MODULE__
 
-  @custom_metadata_url "https://archive.org/download/{identifier}/metadata.json"
   @archive_metadata_url "http://archive.org/metadata/{identifier}"
   @download_url "https://archive.org/download/{identifier}/{filename}"
   @podcast_link "https://archive.org/details/{identifier}"
@@ -30,7 +29,6 @@ defmodule PodcastFeed.Provider.Archive.Parser do
   def by_identifier(identifier) do
     %Parser{identifier: identifier}
     |> enrich_with_archive_metadata()
-    |> enrich_with_custom_metadata()
     |> to_podcast_feed_data()
   end
 
@@ -41,7 +39,7 @@ defmodule PodcastFeed.Provider.Archive.Parser do
     }
   end
 
-  defp podcast_data(token = %{custom_metadata: custom, archive_metadata: %{"metadata" => metadata, "item_last_updated" => last_updated}}) do
+  defp podcast_data(token = %{archive_metadata: %{"metadata" => metadata, "item_last_updated" => last_updated}}) do
     link = Format.compile(@podcast_link, identifier: token.identifier)
     %{
       title: metadata["title"],
@@ -58,13 +56,13 @@ defmodule PodcastFeed.Provider.Archive.Parser do
       author: metadata["creator"],
       language: metadata["language"],
       image: %{
-        url: get_in(custom, ["image", "url"]) || fetch_cover(token),
+        url: fetch_cover(token),
-        title: get_in(custom, ["image", "title"]) || metadata["title"],
+        title: metadata["title"],
-        link: get_in(custom, ["image", "link"]) || link,
+        link: Map.get(metadata, "op_link") || link,
       },
-      link: Map.get(custom, "link") || link,
+      link: Map.get(metadata, "op_link") || link,
-      category: Map.get(custom, "category", ""),
+      category: Map.get(metadata, "op_category", ""),
-      explicit: Map.get(custom, "explicit", "no"),
+      explicit: Map.get(metadata, "op_explicit", "no"),
     }
   end
 
@@ -74,20 +72,6 @@ defmodule PodcastFeed.Provider.Archive.Parser do
     |> Enum.map(fn f -> to_feed_item(f, identifier, files) end)
   end
 
-  defp fetch_custom_metadata(identifier) do
-    custom_metadata_url = Format.compile(@custom_metadata_url, identifier: identifier)
-    parse_custom_metadata_response(:hackney.get(custom_metadata_url, [], "", [follow_redirect: true]))
-  end
-
-  defp parse_custom_metadata_response({:ok, 200, _headers, client_ref}) do
-    {:ok, custom_metadata_json} = :hackney.body(client_ref)
-    custom_metadata_json
-    |> String.split("\n")
-    |> Enum.join()
-    |> Jason.decode!()
-  end
-  defp parse_custom_metadata_response(_), do: @custom_metadata_defaults
-
   defp fetch_archive_metadata(identifier) do
     metadata_url = Format.compile(@archive_metadata_url, identifier: identifier)
     {:ok, 200, _headers, client_ref} = :hackney.get(metadata_url, [], "", [follow_redirect: true, connect_timeout: 30_000, recv_timeout: 30_000])
@@ -152,10 +136,6 @@ defmodule PodcastFeed.Provider.Archive.Parser do
     %Parser{token | archive_metadata: fetch_archive_metadata(token.identifier)}
   end
 
-  defp enrich_with_custom_metadata(token) do
-    %Parser{token | custom_metadata: fetch_custom_metadata(token.identifier)}
-  end
-
   defp parse_subject(subject) when is_list(subject), do: subject
   defp parse_subject(subject) when is_binary(subject), do: subject |> String.split(";")
 end

test/podcast_feed/provider/archive/parser_test.exs

@@ -4,13 +4,11 @@ defmodule PodcastFeed.Provider.Archive.ParserTest do
   alias PodcastFeed.Provider.Archive.Parser
 
   setup_all do
-    json_metadata = "{\"created\":1590247789,\"d1\":\"ia601402.us.archive.org\",\"d2\":\"ia801402.us.archive.org\",\"dir\":\"/8/items/incontri-a-piano-terra\",\"files\":[{\"name\": \"metadata.json\",\"source\": \"original\",\"mtime\": \"1590258296\",\"size\": \"315\",\"md5\": \"a0c0e219cf3f13e54f2a4b3efef8e5c8\",\"crc32\": \"2d181b5c\",\"sha1\": \"8244b579a759edddd01c905dd11f8565e83d0898\",\"format\": \"JSON\"},{\"name\": \"cover.jpg\",\"source\": \"original\",\"mtime\": \"1590258445\",\"size\": \"10650\",\"md5\": \"15687b23e11f0099abbfe64eb1685c31\",\"crc32\": \"fbb1516a\",\"sha1\": \"98fa929c7554241cfa92bea8eba69b39c5d47603\",\"format\": \"JPEG\",\"rotation\": \"0\"},{\"name\":\"Confini mobili sulle alpi.mp3\",\"source\":\"original\",\"mtime\":\"1590135989\",\"size\":\"46933494\",\"md5\":\"e832ee9381a4f8af2d9727e2f49126ae\",\"crc32\":\"d709dd90\",\"sha1\":\"89c820a2dfd63cfbbf7aeefd191c653756b33fe3\",\"format\":\"VBR MP3\",\"length\":\"3902.35\",\"height\":\"0\",\"width\":\"0\",\"title\":\"Confini mobili sulle alpi (italian limes)\",\"creator\":\"APE Milano\",\"album\":\"Incontri a Piano Terra\",\"track\":\"02\",\"artist\":\"APE Milano\",\"genre\":\"podcast\"},{\"name\":\"Confini mobili sulle alpi.png\",\"source\":\"derivative\",\"format\":\"PNG\",\"original\":\"Confini mobili sulle alpi.mp3\",\"mtime\":\"1590137809\",\"size\":\"34656\",\"md5\":\"63893f9b00402a107682b5317e808523\",\"crc32\":\"b59ff609\",\"sha1\":\"a396716431cd0acedd243030093d0b31d792cfb3\"},{\"name\":\"Confini mobili sulle alpi_spectrogram.png\",\"source\":\"derivative\",\"format\":\"Spectrogram\",\"original\":\"Confini mobili sulle alpi.mp3\",\"mtime\":\"1590137854\",\"size\":\"273188\",\"md5\":\"557337665c6d9f962b2e91d169f25e1b\",\"crc32\":\"08b4b57c\",\"sha1\":\"88e088f9c4954aa8f0849b7e0d69cee8d7d42327\"}],\"files_count\":31,\"item_last_updated\":1590160774,\"item_size\":244544362,\"metadata\":{\"identifier\":\"incontri-a-piano-terra\",\"mediatype\":\"audio\",\"collection\":\"opensource_audio\",\"creator\":\"APE Milano\",\"description\":\"Qualche registrazione delle attivit\\u00e0 sociali che promuoviamo al Piano Terra di Milano\",\"language\":\"ita\",\"licenseurl\":\"https://creativecommons.org/licenses/by-nc-nd/4.0/\",\"scanner\":\"Internet Archive HTML5 Uploader 1.6.4\",\"subject\":[\"ape milano\",\"podcast\",\"montagna\"],\"title\":\"Incontri a Piano Terra\",\"uploader\":\"milanoape@gmail.com\",\"publicdate\":\"2020-05-22 08:30:21\",\"addeddate\":\"2020-05-22 08:30:21\",\"curation\":\"[curator]validator@archive.org[/curator][date]20200522085526[/date][comment]checked for malware[/comment]\"},\"server\":\"ia601402.us.archive.org\",\"uniq\":122833277,\"workable_servers\":[\"ia601402.us.archive.org\",\"ia801402.us.archive.org\"]}"
+    json_metadata = "{\"created\":1590247789,\"d1\":\"ia601402.us.archive.org\",\"d2\":\"ia801402.us.archive.org\",\"dir\":\"/8/items/incontri-a-piano-terra\",\"files\":[{\"name\": \"metadata.json\",\"source\": \"original\",\"mtime\": \"1590258296\",\"size\": \"315\",\"md5\": \"a0c0e219cf3f13e54f2a4b3efef8e5c8\",\"crc32\": \"2d181b5c\",\"sha1\": \"8244b579a759edddd01c905dd11f8565e83d0898\",\"format\": \"JSON\"},{\"name\": \"cover.jpg\",\"source\": \"original\",\"mtime\": \"1590258445\",\"size\": \"10650\",\"md5\": \"15687b23e11f0099abbfe64eb1685c31\",\"crc32\": \"fbb1516a\",\"sha1\": \"98fa929c7554241cfa92bea8eba69b39c5d47603\",\"format\": \"JPEG\",\"rotation\": \"0\"},{\"name\":\"Confini mobili sulle alpi.mp3\",\"source\":\"original\",\"mtime\":\"1590135989\",\"size\":\"46933494\",\"md5\":\"e832ee9381a4f8af2d9727e2f49126ae\",\"crc32\":\"d709dd90\",\"sha1\":\"89c820a2dfd63cfbbf7aeefd191c653756b33fe3\",\"format\":\"VBR MP3\",\"length\":\"3902.35\",\"height\":\"0\",\"width\":\"0\",\"title\":\"Confini mobili sulle alpi (italian limes)\",\"creator\":\"APE Milano\",\"album\":\"Incontri a Piano Terra\",\"track\":\"02\",\"artist\":\"APE Milano\",\"genre\":\"podcast\"},{\"name\":\"Confini mobili sulle alpi.png\",\"source\":\"derivative\",\"format\":\"PNG\",\"original\":\"Confini mobili sulle alpi.mp3\",\"mtime\":\"1590137809\",\"size\":\"34656\",\"md5\":\"63893f9b00402a107682b5317e808523\",\"crc32\":\"b59ff609\",\"sha1\":\"a396716431cd0acedd243030093d0b31d792cfb3\"},{\"name\":\"Confini mobili sulle alpi_spectrogram.png\",\"source\":\"derivative\",\"format\":\"Spectrogram\",\"original\":\"Confini mobili sulle alpi.mp3\",\"mtime\":\"1590137854\",\"size\":\"273188\",\"md5\":\"557337665c6d9f962b2e91d169f25e1b\",\"crc32\":\"08b4b57c\",\"sha1\":\"88e088f9c4954aa8f0849b7e0d69cee8d7d42327\"}],\"files_count\":31,\"item_last_updated\":1590160774,\"item_size\":244544362,\"metadata\":{\"identifier\":\"incontri-a-piano-terra\",\"mediatype\":\"audio\",\"collection\":\"opensource_audio\",\"creator\":\"APE Milano\",\"description\":\"Qualche registrazione delle attivit\\u00e0 sociali che promuoviamo al Piano Terra di Milano\",\"language\":\"ita\",\"licenseurl\":\"https://creativecommons.org/licenses/by-nc-nd/4.0/\",\"scanner\":\"Internet Archive HTML5 Uploader 1.6.4\",\"subject\":[\"ape milano\",\"podcast\",\"montagna\"],\"title\":\"Incontri a Piano Terra\",\"uploader\":\"milanoape@gmail.com\",\"publicdate\":\"2020-05-22 08:30:21\",\"addeddate\":\"2020-05-22 08:30:21\",\"curation\":\"[curator]validator@archive.org[/curator][date]20200522085526[/date][comment]checked for malware[/comment]\",\"op_link\":\"http://www.ape-alveare.it/\",\"op_category\":\"Montagna\",\"op_explicit\":\"no\"},\"server\":\"ia601402.us.archive.org\",\"uniq\":122833277,\"workable_servers\":[\"ia601402.us.archive.org\",\"ia801402.us.archive.org\"]}"
-    json_custom = "{\"link\": \"http://www.ape-alveare.it/\",\"image\": {\"url\": \"http://www.ape-alveare.it/wp-content/themes/yootheme/cache/2018_logo_Ape_righe_trasparenza-d1aae6b9.png\",\"title\": \"APE Milano\",\"link\": \"http://www.ape-alveare.it/\"},\"category\": \"Montagna\",\"explicit\": \"no\" }"
 
     token = %Parser{
       identifier: "incontri-a-piano-terra",
       archive_metadata: Jason.decode!(json_metadata),
-      custom_metadata: Jason.decode!(json_custom),
     }
 
     {:ok, token: token}
@@ -34,9 +32,9 @@ defmodule PodcastFeed.Provider.Archive.ParserTest do
       author: "APE Milano",
       language: "ita",
       image: %{
-        url: "http://www.ape-alveare.it/wp-content/themes/yootheme/cache/2018_logo_Ape_righe_trasparenza-d1aae6b9.png",
-        title: "APE Milano",
         link: "http://www.ape-alveare.it/",
+        title: "Incontri a Piano Terra",
+        url: "https://archive.org/download/incontri-a-piano-terra/cover.jpg"
       },
       link: "http://www.ape-alveare.it/",
       category: "Montagna",
@@ -53,30 +51,6 @@ defmodule PodcastFeed.Provider.Archive.ParserTest do
     } = podcast
   end
 
-  test "using first `original` image found on archive when image is missing on custom metadata", state do
-    custom = %{state[:token].custom_metadata | "image" => %{
-      "url" => nil,
-      "title" => nil,
-      "link" => nil,
-    }}
-    %{podcast: podcast} = Parser.to_podcast_feed_data(%Parser{state[:token] | custom_metadata: custom})
-
-    assert %{
-      image: %{
-        url: "https://archive.org/download/incontri-a-piano-terra/cover.jpg",
-        title: "Incontri a Piano Terra",
-        link: "https://archive.org/details/incontri-a-piano-terra",
-      },
-    } = podcast
-  end
-
-  test "using archive item details as link when link is missing on custom metadata", state do
-    custom = %{state[:token].custom_metadata | "link" => nil}
-    %{podcast: podcast} = Parser.to_podcast_feed_data(%Parser{state[:token] | custom_metadata: custom})
-
-    assert %{link: "https://archive.org/details/incontri-a-piano-terra"} = podcast
-  end
-
   test "items data are correctly converted", state do
     %{items: items} = Parser.to_podcast_feed_data(state[:token])