From b1614e92672ceda688e4269b327ef874fff2f6f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20L=C5=93uillet?= Date: Sat, 24 May 2025 16:23:38 +0200 Subject: [PATCH 01/11] Add Pocket and Shaarli imports --- app/config/config.yml | 28 ++ app/config/services.yml | 18 ++ app/config/services_rabbit.yml | 12 + app/config/services_redis.yml | 32 +++ app/config/wallabag.yml | 2 +- phpstan-baseline.neon | 10 + .../ImportBundle/Command/ImportCommand.php | 39 ++- .../Consumer/RabbitMQConsumerTotalProxy.php | 27 +- .../Controller/HtmlController.php | 83 ++++++ .../Controller/ImportController.php | 4 + .../Controller/PocketHtmlController.php | 57 ++++ .../Controller/ShaarliController.php | 57 ++++ .../ImportBundle/Import/HtmlImport.php | 210 +++++++++++++++ .../ImportBundle/Import/PocketHtmlImport.php | 113 ++++++++ .../ImportBundle/Import/ShaarliImport.php | 66 +++++ .../views/PocketHtml/index.html.twig | 45 ++++ .../Resources/views/Shaarli/index.html.twig | 45 ++++ .../Controller/FirefoxControllerTest.php | 6 +- .../Controller/ImportControllerTest.php | 2 +- .../Controller/PocketHtmlControllerTest.php | 168 ++++++++++++ .../Controller/ShaarliControllerTest.php | 168 ++++++++++++ .../Import/PocketHtmlImportTest.php | 254 ++++++++++++++++++ .../ImportBundle/Import/ShaarliImportTest.php | 254 ++++++++++++++++++ .../ImportBundle/fixtures/ril_export.html | 21 ++ .../fixtures/shaarli-bookmarks.html | 13 + translations/messages.en.yml | 8 + 26 files changed, 1732 insertions(+), 10 deletions(-) create mode 100644 src/Wallabag/ImportBundle/Controller/HtmlController.php create mode 100644 src/Wallabag/ImportBundle/Controller/PocketHtmlController.php create mode 100644 src/Wallabag/ImportBundle/Controller/ShaarliController.php create mode 100644 src/Wallabag/ImportBundle/Import/HtmlImport.php create mode 100644 src/Wallabag/ImportBundle/Import/PocketHtmlImport.php create mode 100644 src/Wallabag/ImportBundle/Import/ShaarliImport.php create mode 100644 src/Wallabag/ImportBundle/Resources/views/PocketHtml/index.html.twig create mode 100644 src/Wallabag/ImportBundle/Resources/views/Shaarli/index.html.twig create mode 100644 tests/Wallabag/ImportBundle/Controller/PocketHtmlControllerTest.php create mode 100644 tests/Wallabag/ImportBundle/Controller/ShaarliControllerTest.php create mode 100644 tests/Wallabag/ImportBundle/Import/PocketHtmlImportTest.php create mode 100644 tests/Wallabag/ImportBundle/Import/ShaarliImportTest.php create mode 100644 tests/Wallabag/ImportBundle/fixtures/ril_export.html create mode 100644 tests/Wallabag/ImportBundle/fixtures/shaarli-bookmarks.html diff --git a/app/config/config.yml b/app/config/config.yml index cd981163d..1a522ddbe 100644 --- a/app/config/config.yml +++ b/app/config/config.yml @@ -283,6 +283,16 @@ old_sound_rabbit_mq: exchange_options: name: 'wallabag.import.chrome' type: topic + import_shaarli: + connection: default + exchange_options: + name: 'wallabag.import.shaarli' + type: topic + import_pocket_html: + connection: default + exchange_options: + name: 'wallabag.import.pocket_html' + type: topic consumers: import_pocket: connection: default @@ -383,6 +393,24 @@ old_sound_rabbit_mq: name: 'wallabag.import.chrome' callback: wallabag_import.consumer.amqp.chrome qos_options: {prefetch_count: "%rabbitmq_prefetch_count%"} + import_shaarli: + connection: default + exchange_options: + name: 'wallabag.import.shaarli' + type: topic + queue_options: + name: 'wallabag.import.shaarli' + callback: wallabag_import.consumer.amqp.shaarli + qos_options: {prefetch_count: "%rabbitmq_prefetch_count%"} + import_pocket_html: + connection: default + exchange_options: + name: 'wallabag.import.pocket_html' + type: topic + queue_options: + name: 'wallabag.import.pocket_html' + callback: wallabag_import.consumer.amqp.pocket_html + qos_options: {prefetch_count: "%rabbitmq_prefetch_count%"} fos_js_routing: routes_to_expose: diff --git a/app/config/services.yml b/app/config/services.yml index 47f9b3aef..cc7c5329c 100644 --- a/app/config/services.yml +++ b/app/config/services.yml @@ -121,6 +121,16 @@ services: $rabbitMqProducer: '@old_sound_rabbit_mq.import_wallabag_v2_producer' $redisProducer: '@wallabag_import.producer.redis.wallabag_v2' + Wallabag\ImportBundle\Controller\ShaarliController: + arguments: + $rabbitMqProducer: '@old_sound_rabbit_mq.import_shaarli_producer' + $redisProducer: '@wallabag_import.producer.redis.shaarli' + + Wallabag\ImportBundle\Controller\PocketHtmlController: + arguments: + $rabbitMqProducer: '@old_sound_rabbit_mq.import_pocket_html_producer' + $redisProducer: '@wallabag_import.producer.redis.pocket_html' + Wallabag\ImportBundle\: resource: '../../src/Wallabag/ImportBundle/*' exclude: '../../src/Wallabag/ImportBundle/{Consumer,Controller,Redis}' @@ -394,6 +404,14 @@ services: tags: - { name: wallabag_import.import, alias: chrome } + Wallabag\ImportBundle\Import\ShaarliImport: + tags: + - { name: wallabag_import.import, alias: shaarli } + + Wallabag\ImportBundle\Import\PocketHtmlImport: + tags: + - { name: wallabag_import.import, alias: pocket_html } + # to factorize the proximity and bypass translation for prev & next pagerfanta.view.default_wallabag: class: Pagerfanta\View\OptionableView diff --git a/app/config/services_rabbit.yml b/app/config/services_rabbit.yml index b6f9f1d52..e6963733a 100644 --- a/app/config/services_rabbit.yml +++ b/app/config/services_rabbit.yml @@ -19,6 +19,8 @@ services: $deliciousConsumer: '@old_sound_rabbit_mq.import_delicious_consumer' $elcuratorConsumer: '@old_sound_rabbit_mq.import_elcurator_consumer' $omnivoreConsumer: '@old_sound_rabbit_mq.import_omnivore_consumer' + $shaarliConsumer: '@old_sound_rabbit_mq.import_shaarli_consumer' + $pocketHtmlConsumer: '@old_sound_rabbit_mq.import_pocket_html_consumer' wallabag_import.consumer.amqp.pocket: class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer @@ -74,3 +76,13 @@ services: class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer arguments: $import: '@Wallabag\ImportBundle\Import\ChromeImport' + + wallabag_import.consumer.amqp.shaarli: + class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer + arguments: + $import: '@Wallabag\ImportBundle\Import\ShaarliImport' + + wallabag_import.consumer.amqp.pocket_html: + class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer + arguments: + $import: '@Wallabag\ImportBundle\Import\PocketHtmlImport' diff --git a/app/config/services_redis.yml b/app/config/services_redis.yml index 2f0f1ac48..baeb1ccef 100644 --- a/app/config/services_redis.yml +++ b/app/config/services_redis.yml @@ -180,3 +180,35 @@ services: class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer arguments: $import: '@Wallabag\ImportBundle\Import\ChromeImport' + + # shaarli + wallabag_import.queue.redis.shaarli: + class: Simpleue\Queue\RedisQueue + arguments: + $queueName: "wallabag.import.shaarli" + + wallabag_import.producer.redis.shaarli: + class: Wallabag\ImportBundle\Redis\Producer + arguments: + - "@wallabag_import.queue.redis.shaarli" + + wallabag_import.consumer.redis.shaarli: + class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer + arguments: + $import: '@Wallabag\ImportBundle\Import\ShaarliImport' + + # pocket html + wallabag_import.queue.redis.pocket_html: + class: Simpleue\Queue\RedisQueue + arguments: + $queueName: "wallabag.import.pocket_html" + + wallabag_import.producer.redis.pocket_html: + class: Wallabag\ImportBundle\Redis\Producer + arguments: + - "@wallabag_import.queue.redis.pocket_html" + + wallabag_import.consumer.redis.pocket_html: + class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer + arguments: + $import: '@Wallabag\ImportBundle\Import\PocketHtmlImport' diff --git a/app/config/wallabag.yml b/app/config/wallabag.yml index 09d12619b..2c5e53a00 100644 --- a/app/config/wallabag.yml +++ b/app/config/wallabag.yml @@ -167,5 +167,5 @@ wallabag_core: rule: _all ~ "https?://www\.lemonde\.fr/tiny.*" wallabag_import: - allow_mimetypes: ['application/octet-stream', 'application/json', 'text/plain', 'text/csv'] + allow_mimetypes: ['application/octet-stream', 'application/json', 'text/plain', 'text/csv', 'text/html'] resource_dir: "%kernel.project_dir%/web/uploads/import" diff --git a/phpstan-baseline.neon b/phpstan-baseline.neon index ae5029a61..1e423b482 100644 --- a/phpstan-baseline.neon +++ b/phpstan-baseline.neon @@ -59,3 +59,13 @@ parameters: message: "#^Method FOS\\\\UserBundle\\\\Model\\\\UserManagerInterface\\:\\:updateUser()#" count: 7 path: src/Wallabag/CoreBundle/Controller/ConfigController.php + + - + message: "#^Call to an undefined method Wallabag\\\\ImportBundle\\\\Import\\\\ImportInterface\\:\\:setUser\\(\\)\\.$#" + count: 1 + path: src/Wallabag/ImportBundle/Controller/HtmlController.php + + - + message: "#^Call to an undefined method Wallabag\\\\ImportBundle\\\\Import\\\\ImportInterface\\:\\:setFilepath\\(\\)\\.$#" + count: 1 + path: src/Wallabag/ImportBundle/Controller/HtmlController.php diff --git a/src/Wallabag/ImportBundle/Command/ImportCommand.php b/src/Wallabag/ImportBundle/Command/ImportCommand.php index edc85c01d..76bd4f451 100644 --- a/src/Wallabag/ImportBundle/Command/ImportCommand.php +++ b/src/Wallabag/ImportBundle/Command/ImportCommand.php @@ -13,11 +13,14 @@ use Symfony\Component\Security\Core\Authentication\Token\Storage\TokenStorageInt use Symfony\Component\Security\Core\Authentication\Token\UsernamePasswordToken; use Wallabag\ImportBundle\Import\ChromeImport; use Wallabag\ImportBundle\Import\DeliciousImport; +use Wallabag\ImportBundle\Import\ElcuratorImport; use Wallabag\ImportBundle\Import\FirefoxImport; use Wallabag\ImportBundle\Import\InstapaperImport; use Wallabag\ImportBundle\Import\OmnivoreImport; use Wallabag\ImportBundle\Import\PinboardImport; +use Wallabag\ImportBundle\Import\PocketHtmlImport; use Wallabag\ImportBundle\Import\ReadabilityImport; +use Wallabag\ImportBundle\Import\ShaarliImport; use Wallabag\ImportBundle\Import\WallabagV1Import; use Wallabag\ImportBundle\Import\WallabagV2Import; use Wallabag\UserBundle\Entity\User; @@ -37,9 +40,27 @@ class ImportCommand extends Command private DeliciousImport $deliciousImport; private OmnivoreImport $omnivoreImport; private WallabagV1Import $wallabagV1Import; + private ElcuratorImport $elcuratorImport; + private ShaarliImport $shaarliImport; + private PocketHtmlImport $pocketHtmlImport; - public function __construct(EntityManagerInterface $entityManager, TokenStorageInterface $tokenStorage, UserRepository $userRepository, WallabagV2Import $wallabagV2Import, FirefoxImport $firefoxImport, ChromeImport $chromeImport, ReadabilityImport $readabilityImport, InstapaperImport $instapaperImport, PinboardImport $pinboardImport, DeliciousImport $deliciousImport, OmnivoreImport $omnivoreImport, WallabagV1Import $wallabagV1Import) - { + public function __construct( + EntityManagerInterface $entityManager, + TokenStorageInterface $tokenStorage, + UserRepository $userRepository, + WallabagV2Import $wallabagV2Import, + FirefoxImport $firefoxImport, + ChromeImport $chromeImport, + ReadabilityImport $readabilityImport, + InstapaperImport $instapaperImport, + PinboardImport $pinboardImport, + DeliciousImport $deliciousImport, + WallabagV1Import $wallabagV1Import, + ElcuratorImport $elcuratorImport, + OmnivoreImport $omnivoreImport, + ShaarliImport $shaarliImport, + PocketHtmlImport $pocketHtmlImport + ) { $this->entityManager = $entityManager; $this->tokenStorage = $tokenStorage; $this->userRepository = $userRepository; @@ -52,6 +73,9 @@ class ImportCommand extends Command $this->deliciousImport = $deliciousImport; $this->omnivoreImport = $omnivoreImport; $this->wallabagV1Import = $wallabagV1Import; + $this->elcuratorImport = $elcuratorImport; + $this->shaarliImport = $shaarliImport; + $this->pocketHtmlImport = $pocketHtmlImport; parent::__construct(); } @@ -63,7 +87,7 @@ class ImportCommand extends Command ->setDescription('Import entries from a JSON export') ->addArgument('username', InputArgument::REQUIRED, 'User to populate') ->addArgument('filepath', InputArgument::REQUIRED, 'Path to the JSON file') - ->addOption('importer', null, InputOption::VALUE_OPTIONAL, 'The importer to use: v1, v2, instapaper, pinboard, delicious, readability, firefox or chrome', 'v1') + ->addOption('importer', null, InputOption::VALUE_OPTIONAL, 'The importer to use: v1, v2, instapaper, pinboard, delicious, readability, firefox, chrome, elcurator, shaarli or pocket', 'v1') ->addOption('markAsRead', null, InputOption::VALUE_OPTIONAL, 'Mark all entries as read', false) ->addOption('useUserId', null, InputOption::VALUE_NONE, 'Use user id instead of username to find account') ->addOption('disableContentUpdate', null, InputOption::VALUE_NONE, 'Disable fetching updated content from URL') @@ -125,6 +149,15 @@ class ImportCommand extends Command break; case 'omnivore': $import = $this->omnivoreImport; + break; + case 'elcurator': + $import = $this->elcuratorImport; + break; + case 'shaarli': + $import = $this->shaarliImport; + break; + case 'pocket': + $import = $this->pocketHtmlImport; break; default: $import = $this->wallabagV1Import; diff --git a/src/Wallabag/ImportBundle/Consumer/RabbitMQConsumerTotalProxy.php b/src/Wallabag/ImportBundle/Consumer/RabbitMQConsumerTotalProxy.php index 096d2b9a6..0d531d884 100644 --- a/src/Wallabag/ImportBundle/Consumer/RabbitMQConsumerTotalProxy.php +++ b/src/Wallabag/ImportBundle/Consumer/RabbitMQConsumerTotalProxy.php @@ -21,9 +21,24 @@ class RabbitMQConsumerTotalProxy private Consumer $deliciousConsumer; private Consumer $elcuratorConsumer; private Consumer $omnivoreConsumer; + private Consumer $shaarliConsumer; + private Consumer $pocketHtmlConsumer; - public function __construct(Consumer $pocketConsumer, Consumer $readabilityConsumer, Consumer $wallabagV1Consumer, Consumer $wallabagV2Consumer, Consumer $firefoxConsumer, Consumer $chromeConsumer, Consumer $instapaperConsumer, Consumer $pinboardConsumer, Consumer $deliciousConsumer, Consumer $elcuratorConsumer, Consumer $omnivoreConsumer) - { + public function __construct( + Consumer $pocketConsumer, + Consumer $readabilityConsumer, + Consumer $wallabagV1Consumer, + Consumer $wallabagV2Consumer, + Consumer $firefoxConsumer, + Consumer $chromeConsumer, + Consumer $instapaperConsumer, + Consumer $pinboardConsumer, + Consumer $deliciousConsumer, + Consumer $elcuratorConsumer, + Consumer $omnivoreConsumer, + Consumer $shaarliConsumer, + Consumer $pocketHtmlConsumer + ) { $this->pocketConsumer = $pocketConsumer; $this->readabilityConsumer = $readabilityConsumer; $this->wallabagV1Consumer = $wallabagV1Consumer; @@ -35,6 +50,8 @@ class RabbitMQConsumerTotalProxy $this->deliciousConsumer = $deliciousConsumer; $this->elcuratorConsumer = $elcuratorConsumer; $this->omnivoreConsumer = $omnivoreConsumer; + $this->shaarliConsumer = $shaarliConsumer; + $this->pocketHtmlConsumer = $pocketHtmlConsumer; } /** @@ -82,6 +99,12 @@ class RabbitMQConsumerTotalProxy case 'omnivore': $consumer = $this->omnivoreConsumer; break; + case 'shaarli': + $consumer = $this->shaarliConsumer; + break; + case 'pocket_html': + $consumer = $this->pocketHtmlConsumer; + break; default: return 0; } diff --git a/src/Wallabag/ImportBundle/Controller/HtmlController.php b/src/Wallabag/ImportBundle/Controller/HtmlController.php new file mode 100644 index 000000000..e9515561f --- /dev/null +++ b/src/Wallabag/ImportBundle/Controller/HtmlController.php @@ -0,0 +1,83 @@ +createForm(UploadImportType::class); + $form->handleRequest($request); + + $wallabag = $this->getImportService(); + $wallabag->setUser($this->getUser()); + + if ($form->isSubmitted() && $form->isValid()) { + $file = $form->get('file')->getData(); + $markAsRead = $form->get('mark_as_read')->getData(); + $name = $this->getUser()->getId() . '.html'; + + if (null !== $file && \in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes'), true) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) { + $res = $wallabag + ->setFilepath($this->getParameter('wallabag_import.resource_dir') . '/' . $name) + ->setMarkAsRead($markAsRead) + ->import(); + + $message = 'flashes.import.notice.failed'; + + if (true === $res) { + $summary = $wallabag->getSummary(); + $message = $translator->trans('flashes.import.notice.summary', [ + '%imported%' => $summary['imported'], + '%skipped%' => $summary['skipped'], + ]); + + if (0 < $summary['queued']) { + $message = $translator->trans('flashes.import.notice.summary_with_queue', [ + '%queued%' => $summary['queued'], + ]); + } + + unlink($this->getParameter('wallabag_import.resource_dir') . '/' . $name); + } + + $this->addFlash('notice', $message); + + return $this->redirect($this->generateUrl('homepage')); + } + $this->addFlash('notice', 'flashes.import.notice.failed_on_file'); + } + + return $this->render($this->getImportTemplate(), [ + 'form' => $form->createView(), + 'import' => $wallabag, + ]); + } + + /** + * Return the service to handle the import. + * + * @return ImportInterface + */ + abstract protected function getImportService(); + + /** + * Return the template used for the form. + * + * @return string + */ + abstract protected function getImportTemplate(); +} diff --git a/src/Wallabag/ImportBundle/Controller/ImportController.php b/src/Wallabag/ImportBundle/Controller/ImportController.php index 16414d885..6e6e5d7c1 100644 --- a/src/Wallabag/ImportBundle/Controller/ImportController.php +++ b/src/Wallabag/ImportBundle/Controller/ImportController.php @@ -58,6 +58,8 @@ class ImportController extends AbstractController + $this->rabbitMQConsumerTotalProxy->getTotalMessage('delicious') + $this->rabbitMQConsumerTotalProxy->getTotalMessage('elcurator') + $this->rabbitMQConsumerTotalProxy->getTotalMessage('omnivore') + + $this->rabbitMQConsumerTotalProxy->getTotalMessage('shaarli') + + $this->rabbitMQConsumerTotalProxy->getTotalMessage('pocket_html') ; } catch (\Exception $e) { $rabbitNotInstalled = true; @@ -77,6 +79,8 @@ class ImportController extends AbstractController + $redis->llen('wallabag.import.delicious') + $redis->llen('wallabag.import.elcurator') + $redis->llen('wallabag.import.omnivore') + + $redis->llen('wallabag.import.shaarli') + + $redis->llen('wallabag.import.pocket_html') ; } catch (\Exception $e) { $redisNotInstalled = true; diff --git a/src/Wallabag/ImportBundle/Controller/PocketHtmlController.php b/src/Wallabag/ImportBundle/Controller/PocketHtmlController.php new file mode 100644 index 000000000..7387bbfdc --- /dev/null +++ b/src/Wallabag/ImportBundle/Controller/PocketHtmlController.php @@ -0,0 +1,57 @@ +pocketHtmlImport = $pocketHtmlImport; + $this->craueConfig = $craueConfig; + $this->rabbitMqProducer = $rabbitMqProducer; + $this->redisProducer = $redisProducer; + } + + /** + * @Route("/pocket_html", name="import_pocket_html") + */ + public function indexAction(Request $request, TranslatorInterface $translator) + { + return parent::indexAction($request, $translator); + } + + /** + * {@inheritdoc} + */ + protected function getImportService() + { + if ($this->craueConfig->get('import_with_rabbitmq')) { + $this->pocketHtmlImport->setProducer($this->rabbitMqProducer); + } elseif ($this->craueConfig->get('import_with_redis')) { + $this->pocketHtmlImport->setProducer($this->redisProducer); + } + + return $this->pocketHtmlImport; + } + + /** + * {@inheritdoc} + */ + protected function getImportTemplate() + { + return '@WallabagImport/PocketHtml/index.html.twig'; + } +} diff --git a/src/Wallabag/ImportBundle/Controller/ShaarliController.php b/src/Wallabag/ImportBundle/Controller/ShaarliController.php new file mode 100644 index 000000000..46dfd1473 --- /dev/null +++ b/src/Wallabag/ImportBundle/Controller/ShaarliController.php @@ -0,0 +1,57 @@ +shaarliImport = $shaarliImport; + $this->craueConfig = $craueConfig; + $this->rabbitMqProducer = $rabbitMqProducer; + $this->redisProducer = $redisProducer; + } + + /** + * @Route("/shaarli", name="import_shaarli") + */ + public function indexAction(Request $request, TranslatorInterface $translator) + { + return parent::indexAction($request, $translator); + } + + /** + * {@inheritdoc} + */ + protected function getImportService() + { + if ($this->craueConfig->get('import_with_rabbitmq')) { + $this->shaarliImport->setProducer($this->rabbitMqProducer); + } elseif ($this->craueConfig->get('import_with_redis')) { + $this->shaarliImport->setProducer($this->redisProducer); + } + + return $this->shaarliImport; + } + + /** + * {@inheritdoc} + */ + protected function getImportTemplate() + { + return '@WallabagImport/Shaarli/index.html.twig'; + } +} diff --git a/src/Wallabag/ImportBundle/Import/HtmlImport.php b/src/Wallabag/ImportBundle/Import/HtmlImport.php new file mode 100644 index 000000000..a96107e0b --- /dev/null +++ b/src/Wallabag/ImportBundle/Import/HtmlImport.php @@ -0,0 +1,210 @@ +user) { + $this->logger->error('Wallabag HTML Import: user is not defined'); + + return false; + } + + if (!file_exists($this->filepath) || !is_readable($this->filepath)) { + $this->logger->error('Wallabag HTML Import: unable to read file', ['filepath' => $this->filepath]); + + return false; + } + + $html = new \DOMDocument(); + + libxml_use_internal_errors(true); + $html->loadHTMLFile($this->filepath); + $hrefs = $html->getElementsByTagName('a'); + libxml_use_internal_errors(false); + + if (0 === $hrefs->length) { + $this->logger->error('Wallabag HTML: no entries in imported file'); + + return false; + } + + $entries = []; + foreach ($hrefs as $href) { + $entry = []; + $entry['url'] = $href->getAttribute('href'); + $entry['tags'] = $href->getAttribute('tags'); + $entry['created_at'] = $href->getAttribute('add_date'); + $entries[] = $entry; + } + + if ($this->producer) { + $this->parseEntriesForProducer($entries); + + return true; + } + + $this->parseEntries($entries); + + return true; + } + + /** + * Set file path to the html file. + * + * @param string $filepath + */ + public function setFilepath($filepath) + { + $this->filepath = $filepath; + + return $this; + } + + /** + * {@inheritdoc} + */ + public function parseEntry(array $importedEntry) + { + $url = $importedEntry['url']; + + $existingEntry = $this->em + ->getRepository(Entry::class) + ->findByUrlAndUserId($url, $this->user->getId()); + + if (false !== $existingEntry) { + ++$this->skippedEntries; + + return null; + } + + $data = $this->prepareEntry($importedEntry); + + $entry = new Entry($this->user); + $entry->setUrl($data['url']); + $entry->updateArchived($data['is_archived']); + $createdAt = new \DateTime(); + $createdAt->setTimestamp($data['created_at']); + $entry->setCreatedAt($createdAt); + + // update entry with content (in case fetching failed, the given entry will be return) + $this->fetchContent($entry, $data['url'], $data); + + if (\array_key_exists('tags', $data)) { + $this->tagsAssigner->assignTagsToEntry( + $entry, + $data['tags'] + ); + } + + $this->em->persist($entry); + ++$this->importedEntries; + + return $entry; + } + + /** + * Parse and insert all given entries. + */ + protected function parseEntries(array $entries) + { + $i = 1; + $entryToBeFlushed = []; + + foreach ($entries as $importedEntry) { + $entry = $this->parseEntry($importedEntry); + + if (null === $entry) { + continue; + } + + // @see AbstractImport + $entryToBeFlushed[] = $entry; + + // flush every 20 entries + if (0 === ($i % 20)) { + $this->em->flush(); + + foreach ($entryToBeFlushed as $entry) { + $this->eventDispatcher->dispatch(new EntrySavedEvent($entry), EntrySavedEvent::NAME); + } + + $entryToBeFlushed = []; + } + ++$i; + } + + $this->em->flush(); + + if (!empty($entryToBeFlushed)) { + foreach ($entryToBeFlushed as $entry) { + $this->eventDispatcher->dispatch(new EntrySavedEvent($entry), EntrySavedEvent::NAME); + } + } + } + + /** + * Parse entries and send them to the queue. + * It should just be a simple loop on all item, no call to the database should be done + * to speedup queuing. + * + * Faster parse entries for Producer. + * We don't care to make check at this time. They'll be done by the consumer. + */ + protected function parseEntriesForProducer(array $entries) + { + foreach ($entries as $importedEntry) { + if ((array) $importedEntry !== $importedEntry) { + continue; + } + + // set userId for the producer (it won't know which user is connected) + $importedEntry['userId'] = $this->user->getId(); + + if ($this->markAsRead) { + $importedEntry = $this->setEntryAsRead($importedEntry); + } + + ++$this->queuedEntries; + + $this->producer->publish(json_encode($importedEntry)); + } + } + + /** + * {@inheritdoc} + */ + protected function setEntryAsRead(array $importedEntry) + { + $importedEntry['is_archived'] = 1; + + return $importedEntry; + } + + abstract protected function prepareEntry(array $entry = []); +} diff --git a/src/Wallabag/ImportBundle/Import/PocketHtmlImport.php b/src/Wallabag/ImportBundle/Import/PocketHtmlImport.php new file mode 100644 index 000000000..492a1adfc --- /dev/null +++ b/src/Wallabag/ImportBundle/Import/PocketHtmlImport.php @@ -0,0 +1,113 @@ +user) { + $this->logger->error('Pocket HTML Import: user is not defined'); + + return false; + } + + if (!file_exists($this->filepath) || !is_readable($this->filepath)) { + $this->logger->error('Pocket HTML Import: unable to read file', ['filepath' => $this->filepath]); + + return false; + } + + $html = new \DOMDocument(); + + libxml_use_internal_errors(true); + $html->loadHTMLFile($this->filepath); + $hrefs = $html->getElementsByTagName('a'); + libxml_use_internal_errors(false); + + if (0 === $hrefs->length) { + $this->logger->error('Pocket HTML: no entries in imported file'); + + return false; + } + + $entries = []; + foreach ($hrefs as $href) { + $entry = []; + $entry['url'] = $href->getAttribute('href'); + $entry['tags'] = $href->getAttribute('tags'); + $entry['created_at'] = $href->getAttribute('time_added'); + $entries[] = $entry; + } + + if ($this->producer) { + $this->parseEntriesForProducer($entries); + + return true; + } + + $this->parseEntries($entries); + + return true; + } + + /** + * {@inheritdoc} + */ + protected function prepareEntry(array $entry = []) + { + $data = [ + 'title' => '', + 'html' => false, + 'url' => $entry['url'], + 'is_archived' => (int) $this->markAsRead, + 'is_starred' => false, + 'tags' => '', + 'created_at' => $entry['created_at'], + ]; + + if (\array_key_exists('tags', $entry) && '' !== $entry['tags']) { + $data['tags'] = $entry['tags']; + } + + return $data; + } +} diff --git a/src/Wallabag/ImportBundle/Import/ShaarliImport.php b/src/Wallabag/ImportBundle/Import/ShaarliImport.php new file mode 100644 index 000000000..b4c9dc3c3 --- /dev/null +++ b/src/Wallabag/ImportBundle/Import/ShaarliImport.php @@ -0,0 +1,66 @@ + '', + 'html' => false, + 'url' => $entry['url'], + 'is_archived' => (int) $this->markAsRead, + 'is_starred' => false, + 'tags' => '', + 'created_at' => $entry['created_at'], + ]; + + if (\array_key_exists('tags', $entry) && '' !== $entry['tags']) { + $data['tags'] = $entry['tags']; + } + + return $data; + } +} diff --git a/src/Wallabag/ImportBundle/Resources/views/PocketHtml/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/PocketHtml/index.html.twig new file mode 100644 index 000000000..09f2e689f --- /dev/null +++ b/src/Wallabag/ImportBundle/Resources/views/PocketHtml/index.html.twig @@ -0,0 +1,45 @@ +{% extends "@WallabagCore/layout.html.twig" %} + +{% block title %}{{ 'import.pocket_html.page_title'|trans }}{% endblock %} + +{% block content %} +
+
+
+ {% include '@WallabagImport/Import/_information.html.twig' %} + +
+
{{ import.description|trans|raw }}
+

{{ 'import.pocket_html.how_to'|trans }}

+ +
+ {{ form_start(form, {'method': 'POST'}) }} + {{ form_errors(form) }} +
+
+ {{ form_errors(form.file) }} +
+ {{ form.file.vars.label|trans }} + {{ form_widget(form.file) }} +
+
+ +
+
+
+
{{ 'import.form.mark_as_read_title'|trans }}
+ {{ form_widget(form.mark_as_read) }} + {{ form_label(form.mark_as_read) }} +
+
+ + {{ form_widget(form.save, {'attr': {'class': 'btn waves-effect waves-light'}}) }} + + {{ form_rest(form) }} + +
+
+
+
+
+{% endblock %} diff --git a/src/Wallabag/ImportBundle/Resources/views/Shaarli/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Shaarli/index.html.twig new file mode 100644 index 000000000..edb24e468 --- /dev/null +++ b/src/Wallabag/ImportBundle/Resources/views/Shaarli/index.html.twig @@ -0,0 +1,45 @@ +{% extends "@WallabagCore/layout.html.twig" %} + +{% block title %}{{ 'import.shaarli.page_title'|trans }}{% endblock %} + +{% block content %} +
+
+
+ {% include '@WallabagImport/Import/_information.html.twig' %} + +
+
{{ import.description|trans|raw }}
+

{{ 'import.shaarli.how_to'|trans }}

+ +
+ {{ form_start(form, {'method': 'POST'}) }} + {{ form_errors(form) }} +
+
+ {{ form_errors(form.file) }} +
+ {{ form.file.vars.label|trans }} + {{ form_widget(form.file) }} +
+
+ +
+
+
+
{{ 'import.form.mark_as_read_title'|trans }}
+ {{ form_widget(form.mark_as_read) }} + {{ form_label(form.mark_as_read) }} +
+
+ + {{ form_widget(form.save, {'attr': {'class': 'btn waves-effect waves-light'}}) }} + + {{ form_rest(form) }} + +
+
+
+
+
+{% endblock %} diff --git a/tests/Wallabag/ImportBundle/Controller/FirefoxControllerTest.php b/tests/Wallabag/ImportBundle/Controller/FirefoxControllerTest.php index f6dfff4c8..01a7c2d09 100644 --- a/tests/Wallabag/ImportBundle/Controller/FirefoxControllerTest.php +++ b/tests/Wallabag/ImportBundle/Controller/FirefoxControllerTest.php @@ -123,9 +123,9 @@ class FirefoxControllerTest extends WallabagCoreTestCase ); $this->assertInstanceOf(Entry::class, $content); - $this->assertNotEmpty($content->getMimetype(), 'Mimetype for http://lexpansion.lexpress.fr is ok'); - $this->assertNotEmpty($content->getPreviewPicture(), 'Preview picture for http://lexpansion.lexpress.fr is ok'); - $this->assertNotEmpty($content->getLanguage(), 'Language for http://lexpansion.lexpress.fr is ok'); + $this->assertNotEmpty($content->getMimetype(), 'Mimetype for 20minutes.fr is ok'); + $this->assertNotEmpty($content->getPreviewPicture(), 'Preview picture for 20minutes.fr is ok'); + $this->assertNotEmpty($content->getLanguage(), 'Language for 20minutes.fr is ok'); $this->assertCount(3, $content->getTags()); } diff --git a/tests/Wallabag/ImportBundle/Controller/ImportControllerTest.php b/tests/Wallabag/ImportBundle/Controller/ImportControllerTest.php index cf1739841..76b802b6e 100644 --- a/tests/Wallabag/ImportBundle/Controller/ImportControllerTest.php +++ b/tests/Wallabag/ImportBundle/Controller/ImportControllerTest.php @@ -24,6 +24,6 @@ class ImportControllerTest extends WallabagCoreTestCase $crawler = $client->request('GET', '/import/'); $this->assertSame(200, $client->getResponse()->getStatusCode()); - $this->assertSame(11, $crawler->filter('blockquote')->count()); + $this->assertSame(13, $crawler->filter('blockquote')->count()); } } diff --git a/tests/Wallabag/ImportBundle/Controller/PocketHtmlControllerTest.php b/tests/Wallabag/ImportBundle/Controller/PocketHtmlControllerTest.php new file mode 100644 index 000000000..7ae4a247f --- /dev/null +++ b/tests/Wallabag/ImportBundle/Controller/PocketHtmlControllerTest.php @@ -0,0 +1,168 @@ +logInAs('admin'); + $client = $this->getTestClient(); + + $crawler = $client->request('GET', '/import/pocket_html'); + + $this->assertSame(200, $client->getResponse()->getStatusCode()); + $this->assertSame(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); + $this->assertSame(1, $crawler->filter('input[type=file]')->count()); + } + + public function testImportPocketHtmlWithRabbitEnabled() + { + $this->logInAs('admin'); + $client = $this->getTestClient(); + + $client->getContainer()->get(Config::class)->set('import_with_rabbitmq', 1); + + $crawler = $client->request('GET', '/import/pocket_html'); + + $this->assertSame(200, $client->getResponse()->getStatusCode()); + $this->assertSame(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); + $this->assertSame(1, $crawler->filter('input[type=file]')->count()); + + $client->getContainer()->get(Config::class)->set('import_with_rabbitmq', 0); + } + + public function testImportPocketHtmlBadFile() + { + $this->logInAs('admin'); + $client = $this->getTestClient(); + + $crawler = $client->request('GET', '/import/pocket_html'); + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $data = [ + 'upload_import_file[file]' => '', + ]; + + $client->submit($form, $data); + + $this->assertSame(200, $client->getResponse()->getStatusCode()); + } + + public function testImportPocketHtmlWithRedisEnabled() + { + $this->checkRedis(); + $this->logInAs('admin'); + $client = $this->getTestClient(); + $client->getContainer()->get(Config::class)->set('import_with_redis', 1); + + $crawler = $client->request('GET', '/import/pocket_html'); + + $this->assertSame(200, $client->getResponse()->getStatusCode()); + $this->assertSame(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); + $this->assertSame(1, $crawler->filter('input[type=file]')->count()); + + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $file = new UploadedFile(__DIR__ . '/../fixtures/ril_export.html', 'Bookmarks'); + + $data = [ + 'upload_import_file[file]' => $file, + ]; + + $client->submit($form, $data); + + $this->assertSame(302, $client->getResponse()->getStatusCode()); + + $crawler = $client->followRedirect(); + + $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text'])); + $this->assertStringContainsString('flashes.import.notice.summary', $body[0]); + + $this->assertNotEmpty($client->getContainer()->get(Client::class)->lpop('wallabag.import.pocket_html')); + + $client->getContainer()->get(Config::class)->set('import_with_redis', 0); + } + + public function testImportWallabagWithPocketHtmlFile() + { + $this->logInAs('admin'); + $client = $this->getTestClient(); + + $crawler = $client->request('GET', '/import/pocket_html'); + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $file = new UploadedFile(__DIR__ . '/../fixtures/ril_export.html', 'Bookmarks'); + + $data = [ + 'upload_import_file[file]' => $file, + ]; + + $client->submit($form, $data); + + $this->assertSame(302, $client->getResponse()->getStatusCode()); + + $crawler = $client->followRedirect(); + + $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text'])); + $this->assertStringContainsString('flashes.import.notice.summary', $body[0]); + + $content = $client->getContainer() + ->get(EntityManagerInterface::class) + ->getRepository(Entry::class) + ->findByUrlAndUserId( + 'https://www.20minutes.fr/sport/4002755-20220928-tarn-lapins-ravagent-terrain-match-rugby-doit-etre-annule', + $this->getLoggedInUserId() + ); + + $this->assertInstanceOf(Entry::class, $content); + $this->assertNotEmpty($content->getMimetype(), 'Mimetype for 20minutes.fr is ok'); + $this->assertNotEmpty($content->getPreviewPicture(), 'Preview picture for 20minutes.fr is ok'); + $this->assertNotEmpty($content->getLanguage(), 'Language for 20minutes.fr is ok'); + $this->assertCount(3, $content->getTags()); + + $content = $client->getContainer() + ->get(EntityManagerInterface::class) + ->getRepository(Entry::class) + ->findByUrlAndUserId( + 'https://www.lemonde.fr/disparitions/article/2018/07/05/le-journaliste-et-cineaste-claude-lanzmann-est-mort_5326313_3382.html', + $this->getLoggedInUserId() + ); + + $this->assertInstanceOf(Entry::class, $content); + $this->assertNotEmpty($content->getMimetype(), 'Mimetype for https://www.lemonde.fr is ok'); + $this->assertNotEmpty($content->getPreviewPicture(), 'Preview picture for https://www.lemonde.fr is ok'); + $this->assertNotEmpty($content->getLanguage(), 'Language for https://www.lemonde.fr is ok'); + } + + public function testImportWallabagWithEmptyFile() + { + $this->logInAs('admin'); + $client = $this->getTestClient(); + + $crawler = $client->request('GET', '/import/pocket_html'); + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $file = new UploadedFile(__DIR__ . '/../fixtures/test.html', 'test.html'); + + $data = [ + 'upload_import_file[file]' => $file, + ]; + + $client->submit($form, $data); + + $this->assertSame(302, $client->getResponse()->getStatusCode()); + + $crawler = $client->followRedirect(); + + $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text'])); + $this->assertStringContainsString('flashes.import.notice.failed', $body[0]); + } +} diff --git a/tests/Wallabag/ImportBundle/Controller/ShaarliControllerTest.php b/tests/Wallabag/ImportBundle/Controller/ShaarliControllerTest.php new file mode 100644 index 000000000..8bc9ffb9d --- /dev/null +++ b/tests/Wallabag/ImportBundle/Controller/ShaarliControllerTest.php @@ -0,0 +1,168 @@ +logInAs('admin'); + $client = $this->getTestClient(); + + $crawler = $client->request('GET', '/import/shaarli'); + + $this->assertSame(200, $client->getResponse()->getStatusCode()); + $this->assertSame(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); + $this->assertSame(1, $crawler->filter('input[type=file]')->count()); + } + + public function testImportShaarliWithRabbitEnabled() + { + $this->logInAs('admin'); + $client = $this->getTestClient(); + + $client->getContainer()->get(Config::class)->set('import_with_rabbitmq', 1); + + $crawler = $client->request('GET', '/import/shaarli'); + + $this->assertSame(200, $client->getResponse()->getStatusCode()); + $this->assertSame(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); + $this->assertSame(1, $crawler->filter('input[type=file]')->count()); + + $client->getContainer()->get(Config::class)->set('import_with_rabbitmq', 0); + } + + public function testImportShaarliBadFile() + { + $this->logInAs('admin'); + $client = $this->getTestClient(); + + $crawler = $client->request('GET', '/import/shaarli'); + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $data = [ + 'upload_import_file[file]' => '', + ]; + + $client->submit($form, $data); + + $this->assertSame(200, $client->getResponse()->getStatusCode()); + } + + public function testImportShaarliWithRedisEnabled() + { + $this->checkRedis(); + $this->logInAs('admin'); + $client = $this->getTestClient(); + $client->getContainer()->get(Config::class)->set('import_with_redis', 1); + + $crawler = $client->request('GET', '/import/shaarli'); + + $this->assertSame(200, $client->getResponse()->getStatusCode()); + $this->assertSame(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); + $this->assertSame(1, $crawler->filter('input[type=file]')->count()); + + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $file = new UploadedFile(__DIR__ . '/../fixtures/shaarli-bookmarks.html', 'Bookmarks'); + + $data = [ + 'upload_import_file[file]' => $file, + ]; + + $client->submit($form, $data); + + $this->assertSame(302, $client->getResponse()->getStatusCode()); + + $crawler = $client->followRedirect(); + + $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text'])); + $this->assertStringContainsString('flashes.import.notice.summary', $body[0]); + + $this->assertNotEmpty($client->getContainer()->get(Client::class)->lpop('wallabag.import.shaarli')); + + $client->getContainer()->get(Config::class)->set('import_with_redis', 0); + } + + public function testImportWallabagWithShaarliFile() + { + $this->logInAs('admin'); + $client = $this->getTestClient(); + + $crawler = $client->request('GET', '/import/shaarli'); + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $file = new UploadedFile(__DIR__ . '/../fixtures/shaarli-bookmarks.html', 'Bookmarks'); + + $data = [ + 'upload_import_file[file]' => $file, + ]; + + $client->submit($form, $data); + + $this->assertSame(302, $client->getResponse()->getStatusCode()); + + $crawler = $client->followRedirect(); + + $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text'])); + $this->assertStringContainsString('flashes.import.notice.summary', $body[0]); + + $content = $client->getContainer() + ->get(EntityManagerInterface::class) + ->getRepository(Entry::class) + ->findByUrlAndUserId( + 'https://www.20minutes.fr/sport/4002755-20220928-tarn-lapins-ravagent-terrain-match-rugby-doit-etre-annule', + $this->getLoggedInUserId() + ); + + $this->assertInstanceOf(Entry::class, $content); + $this->assertNotEmpty($content->getMimetype(), 'Mimetype for 20minutes.fr is ok'); + $this->assertNotEmpty($content->getPreviewPicture(), 'Preview picture for 20minutes.fr is ok'); + $this->assertNotEmpty($content->getLanguage(), 'Language for 20minutes.fr is ok'); + $this->assertCount(2, $content->getTags()); + + $content = $client->getContainer() + ->get(EntityManagerInterface::class) + ->getRepository(Entry::class) + ->findByUrlAndUserId( + 'https://www.lemonde.fr/disparitions/article/2018/07/05/le-journaliste-et-cineaste-claude-lanzmann-est-mort_5326313_3382.html', + $this->getLoggedInUserId() + ); + + $this->assertInstanceOf(Entry::class, $content); + $this->assertNotEmpty($content->getMimetype(), 'Mimetype for https://www.lemonde.fr is ok'); + $this->assertNotEmpty($content->getPreviewPicture(), 'Preview picture for https://www.lemonde.fr is ok'); + $this->assertNotEmpty($content->getLanguage(), 'Language for https://www.lemonde.fr is ok'); + } + + public function testImportWallabagWithEmptyFile() + { + $this->logInAs('admin'); + $client = $this->getTestClient(); + + $crawler = $client->request('GET', '/import/shaarli'); + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $file = new UploadedFile(__DIR__ . '/../fixtures/test.html', 'test.html'); + + $data = [ + 'upload_import_file[file]' => $file, + ]; + + $client->submit($form, $data); + + $this->assertSame(302, $client->getResponse()->getStatusCode()); + + $crawler = $client->followRedirect(); + + $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text'])); + $this->assertStringContainsString('flashes.import.notice.failed', $body[0]); + } +} diff --git a/tests/Wallabag/ImportBundle/Import/PocketHtmlImportTest.php b/tests/Wallabag/ImportBundle/Import/PocketHtmlImportTest.php new file mode 100644 index 000000000..6ff5e13a0 --- /dev/null +++ b/tests/Wallabag/ImportBundle/Import/PocketHtmlImportTest.php @@ -0,0 +1,254 @@ +getPocketHtmlImport(); + + $this->assertSame('Pocket HTML', $pocketHtmlImport->getName()); + $this->assertNotEmpty($pocketHtmlImport->getUrl()); + $this->assertSame('import.pocket_html.description', $pocketHtmlImport->getDescription()); + } + + public function testImport() + { + $pocketHtmlImport = $this->getPocketHtmlImport(false, 2); + $pocketHtmlImport->setFilepath(__DIR__ . '/../fixtures/ril_export.html'); + + $entryRepo = $this->getMockBuilder(EntryRepository::class) + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->exactly(2)) + ->method('findByUrlAndUserId') + ->willReturn(false); + + $this->em + ->expects($this->any()) + ->method('getRepository') + ->willReturn($entryRepo); + + $entry = $this->getMockBuilder(Entry::class) + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy + ->expects($this->exactly(2)) + ->method('updateEntry') + ->willReturn($entry); + + $res = $pocketHtmlImport->import(); + + $this->assertTrue($res); + $this->assertSame(['skipped' => 0, 'imported' => 2, 'queued' => 0], $pocketHtmlImport->getSummary()); + } + + public function testImportAndMarkAllAsRead() + { + $pocketHtmlImport = $this->getPocketHtmlImport(false, 1); + $pocketHtmlImport->setFilepath(__DIR__ . '/../fixtures/ril_export.html'); + + $entryRepo = $this->getMockBuilder(EntryRepository::class) + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->exactly(2)) + ->method('findByUrlAndUserId') + ->will($this->onConsecutiveCalls(false, true)); + + $this->em + ->expects($this->any()) + ->method('getRepository') + ->willReturn($entryRepo); + + $this->contentProxy + ->expects($this->exactly(1)) + ->method('updateEntry') + ->willReturn(new Entry($this->user)); + + // check that every entry persisted are archived + $this->em + ->expects($this->any()) + ->method('persist') + ->with($this->callback(function ($persistedEntry) { + return (bool) $persistedEntry->isArchived(); + })); + + $res = $pocketHtmlImport + ->setMarkAsRead(true) + ->import(); + + $this->assertTrue($res); + + $this->assertSame(['skipped' => 1, 'imported' => 1, 'queued' => 0], $pocketHtmlImport->getSummary()); + } + + public function testImportWithRabbit() + { + $pocketHtmlImport = $this->getPocketHtmlImport(); + $pocketHtmlImport->setFilepath(__DIR__ . '/../fixtures/ril_export.html'); + + $entryRepo = $this->getMockBuilder(EntryRepository::class) + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->never()) + ->method('findByUrlAndUserId'); + + $this->em + ->expects($this->never()) + ->method('getRepository'); + + $entry = $this->getMockBuilder(Entry::class) + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy + ->expects($this->never()) + ->method('updateEntry'); + + $producer = $this->getMockBuilder(\OldSound\RabbitMqBundle\RabbitMq\Producer::class) + ->disableOriginalConstructor() + ->getMock(); + + $producer + ->expects($this->exactly(2)) + ->method('publish'); + + $pocketHtmlImport->setProducer($producer); + + $res = $pocketHtmlImport->setMarkAsRead(true)->import(); + + $this->assertTrue($res); + $this->assertSame(['skipped' => 0, 'imported' => 0, 'queued' => 2], $pocketHtmlImport->getSummary()); + } + + public function testImportWithRedis() + { + $pocketHtmlImport = $this->getPocketHtmlImport(); + $pocketHtmlImport->setFilepath(__DIR__ . '/../fixtures/ril_export.html'); + + $entryRepo = $this->getMockBuilder(EntryRepository::class) + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->never()) + ->method('findByUrlAndUserId'); + + $this->em + ->expects($this->never()) + ->method('getRepository'); + + $entry = $this->getMockBuilder(Entry::class) + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy + ->expects($this->never()) + ->method('updateEntry'); + + $factory = new RedisMockFactory(); + $redisMock = $factory->getAdapter(Client::class, true); + + $queue = new RedisQueue($redisMock, 'pocket_html'); + $producer = new Producer($queue); + + $pocketHtmlImport->setProducer($producer); + + $res = $pocketHtmlImport->setMarkAsRead(true)->import(); + + $this->assertTrue($res); + $this->assertSame(['skipped' => 0, 'imported' => 0, 'queued' => 2], $pocketHtmlImport->getSummary()); + + $this->assertNotEmpty($redisMock->lpop('pocket_html')); + } + + public function testImportBadFile() + { + $pocketHtmlImport = $this->getPocketHtmlImport(); + $pocketHtmlImport->setFilepath(__DIR__ . '/../fixtures/wallabag-v1.jsonx'); + + $res = $pocketHtmlImport->import(); + + $this->assertFalse($res); + + $records = $this->logHandler->getRecords(); + $this->assertStringContainsString('Pocket HTML Import: unable to read file', $records[0]['message']); + $this->assertSame('ERROR', $records[0]['level_name']); + } + + public function testImportUserNotDefined() + { + $pocketHtmlImport = $this->getPocketHtmlImport(true); + $pocketHtmlImport->setFilepath(__DIR__ . '/../fixtures/ril_export.html'); + + $res = $pocketHtmlImport->import(); + + $this->assertFalse($res); + + $records = $this->logHandler->getRecords(); + $this->assertStringContainsString('Pocket HTML Import: user is not defined', $records[0]['message']); + $this->assertSame('ERROR', $records[0]['level_name']); + } + + private function getPocketHtmlImport($unsetUser = false, $dispatched = 0) + { + $this->user = new User(); + + $this->em = $this->getMockBuilder(EntityManager::class) + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy = $this->getMockBuilder(ContentProxy::class) + ->disableOriginalConstructor() + ->getMock(); + + $this->tagsAssigner = $this->getMockBuilder(TagsAssigner::class) + ->disableOriginalConstructor() + ->getMock(); + + $dispatcher = $this->getMockBuilder(EventDispatcher::class) + ->disableOriginalConstructor() + ->getMock(); + + $dispatcher + ->expects($this->exactly($dispatched)) + ->method('dispatch'); + + $this->logHandler = new TestHandler(); + $logger = new Logger('test', [$this->logHandler]); + + $wallabag = new PocketHtmlImport($this->em, $this->contentProxy, $this->tagsAssigner, $dispatcher, $logger); + + if (false === $unsetUser) { + $wallabag->setUser($this->user); + } + + return $wallabag; + } +} diff --git a/tests/Wallabag/ImportBundle/Import/ShaarliImportTest.php b/tests/Wallabag/ImportBundle/Import/ShaarliImportTest.php new file mode 100644 index 000000000..04f8223dd --- /dev/null +++ b/tests/Wallabag/ImportBundle/Import/ShaarliImportTest.php @@ -0,0 +1,254 @@ +getShaarliImport(); + + $this->assertSame('Shaarli', $shaarliImport->getName()); + $this->assertNotEmpty($shaarliImport->getUrl()); + $this->assertSame('import.shaarli.description', $shaarliImport->getDescription()); + } + + public function testImport() + { + $shaarliImport = $this->getShaarliImport(false, 2); + $shaarliImport->setFilepath(__DIR__ . '/../fixtures/shaarli-bookmarks.html'); + + $entryRepo = $this->getMockBuilder(EntryRepository::class) + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->exactly(2)) + ->method('findByUrlAndUserId') + ->willReturn(false); + + $this->em + ->expects($this->any()) + ->method('getRepository') + ->willReturn($entryRepo); + + $entry = $this->getMockBuilder(Entry::class) + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy + ->expects($this->exactly(2)) + ->method('updateEntry') + ->willReturn($entry); + + $res = $shaarliImport->import(); + + $this->assertTrue($res); + $this->assertSame(['skipped' => 0, 'imported' => 2, 'queued' => 0], $shaarliImport->getSummary()); + } + + public function testImportAndMarkAllAsRead() + { + $shaarliImport = $this->getShaarliImport(false, 1); + $shaarliImport->setFilepath(__DIR__ . '/../fixtures/shaarli-bookmarks.html'); + + $entryRepo = $this->getMockBuilder(EntryRepository::class) + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->exactly(2)) + ->method('findByUrlAndUserId') + ->will($this->onConsecutiveCalls(false, true)); + + $this->em + ->expects($this->any()) + ->method('getRepository') + ->willReturn($entryRepo); + + $this->contentProxy + ->expects($this->exactly(1)) + ->method('updateEntry') + ->willReturn(new Entry($this->user)); + + // check that every entry persisted are archived + $this->em + ->expects($this->any()) + ->method('persist') + ->with($this->callback(function ($persistedEntry) { + return (bool) $persistedEntry->isArchived(); + })); + + $res = $shaarliImport + ->setMarkAsRead(true) + ->import(); + + $this->assertTrue($res); + + $this->assertSame(['skipped' => 1, 'imported' => 1, 'queued' => 0], $shaarliImport->getSummary()); + } + + public function testImportWithRabbit() + { + $shaarliImport = $this->getShaarliImport(); + $shaarliImport->setFilepath(__DIR__ . '/../fixtures/shaarli-bookmarks.html'); + + $entryRepo = $this->getMockBuilder(EntryRepository::class) + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->never()) + ->method('findByUrlAndUserId'); + + $this->em + ->expects($this->never()) + ->method('getRepository'); + + $entry = $this->getMockBuilder(Entry::class) + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy + ->expects($this->never()) + ->method('updateEntry'); + + $producer = $this->getMockBuilder(\OldSound\RabbitMqBundle\RabbitMq\Producer::class) + ->disableOriginalConstructor() + ->getMock(); + + $producer + ->expects($this->exactly(2)) + ->method('publish'); + + $shaarliImport->setProducer($producer); + + $res = $shaarliImport->setMarkAsRead(true)->import(); + + $this->assertTrue($res); + $this->assertSame(['skipped' => 0, 'imported' => 0, 'queued' => 2], $shaarliImport->getSummary()); + } + + public function testImportWithRedis() + { + $shaarliImport = $this->getShaarliImport(); + $shaarliImport->setFilepath(__DIR__ . '/../fixtures/shaarli-bookmarks.html'); + + $entryRepo = $this->getMockBuilder(EntryRepository::class) + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->never()) + ->method('findByUrlAndUserId'); + + $this->em + ->expects($this->never()) + ->method('getRepository'); + + $entry = $this->getMockBuilder(Entry::class) + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy + ->expects($this->never()) + ->method('updateEntry'); + + $factory = new RedisMockFactory(); + $redisMock = $factory->getAdapter(Client::class, true); + + $queue = new RedisQueue($redisMock, 'shaarli'); + $producer = new Producer($queue); + + $shaarliImport->setProducer($producer); + + $res = $shaarliImport->setMarkAsRead(true)->import(); + + $this->assertTrue($res); + $this->assertSame(['skipped' => 0, 'imported' => 0, 'queued' => 2], $shaarliImport->getSummary()); + + $this->assertNotEmpty($redisMock->lpop('shaarli')); + } + + public function testImportBadFile() + { + $shaarliImport = $this->getShaarliImport(); + $shaarliImport->setFilepath(__DIR__ . '/../fixtures/wallabag-v1.jsonx'); + + $res = $shaarliImport->import(); + + $this->assertFalse($res); + + $records = $this->logHandler->getRecords(); + $this->assertStringContainsString('Wallabag HTML Import: unable to read file', $records[0]['message']); + $this->assertSame('ERROR', $records[0]['level_name']); + } + + public function testImportUserNotDefined() + { + $shaarliImport = $this->getShaarliImport(true); + $shaarliImport->setFilepath(__DIR__ . '/../fixtures/shaarli-bookmarks.html'); + + $res = $shaarliImport->import(); + + $this->assertFalse($res); + + $records = $this->logHandler->getRecords(); + $this->assertStringContainsString('Wallabag HTML Import: user is not defined', $records[0]['message']); + $this->assertSame('ERROR', $records[0]['level_name']); + } + + private function getShaarliImport($unsetUser = false, $dispatched = 0) + { + $this->user = new User(); + + $this->em = $this->getMockBuilder(EntityManager::class) + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy = $this->getMockBuilder(ContentProxy::class) + ->disableOriginalConstructor() + ->getMock(); + + $this->tagsAssigner = $this->getMockBuilder(TagsAssigner::class) + ->disableOriginalConstructor() + ->getMock(); + + $dispatcher = $this->getMockBuilder(EventDispatcher::class) + ->disableOriginalConstructor() + ->getMock(); + + $dispatcher + ->expects($this->exactly($dispatched)) + ->method('dispatch'); + + $this->logHandler = new TestHandler(); + $logger = new Logger('test', [$this->logHandler]); + + $wallabag = new ShaarliImport($this->em, $this->contentProxy, $this->tagsAssigner, $dispatcher, $logger); + + if (false === $unsetUser) { + $wallabag->setUser($this->user); + } + + return $wallabag; + } +} diff --git a/tests/Wallabag/ImportBundle/fixtures/ril_export.html b/tests/Wallabag/ImportBundle/fixtures/ril_export.html new file mode 100644 index 000000000..310c092dc --- /dev/null +++ b/tests/Wallabag/ImportBundle/fixtures/ril_export.html @@ -0,0 +1,21 @@ + + + + + + Pocket Export + + +

Unread

+ + +

Read Archive

+ + + diff --git a/tests/Wallabag/ImportBundle/fixtures/shaarli-bookmarks.html b/tests/Wallabag/ImportBundle/fixtures/shaarli-bookmarks.html new file mode 100644 index 000000000..ad401ea74 --- /dev/null +++ b/tests/Wallabag/ImportBundle/fixtures/shaarli-bookmarks.html @@ -0,0 +1,13 @@ + + + +Bookmarks +

Shaarli export of all bookmarks on Mon, 17 Jul 23 14:31:25 +0200

+

+

The Legacy of Firefox OS. In the two years or so since Mozilla… | by Ben Francis | Medium +
In the two years or so since Mozilla announced the end of Firefox OS as a Mozilla-run project, the B2G source code has found its way into a surprising number of commercial products. +
Template Filters — Eleventy +

+ diff --git a/translations/messages.en.yml b/translations/messages.en.yml index 6bb7e0c4e..e5a195bbb 100644 --- a/translations/messages.en.yml +++ b/translations/messages.en.yml @@ -534,6 +534,14 @@ import: page_title: Import > del.icio.us description: This importer will import all your Delicious bookmarks. Since 2021, you can export again your data from it using the export page (https://del.icio.us/export). Choose the "JSON" format and download it (like "delicious_export.2021.02.06_21.10.json"). how_to: Please select your Delicious export and click on the button below to upload and import it. + shaarli: + page_title: Import > Shaarli + description: This importer will import all your Shaarli bookmarks. Just go to the Tools section, then into "Export database", choose your bookmarks and export them. You will obtain a HTML file. + how_to: Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched. + pocket_html: + page_title: Import > Pocket HTML + description: This importer will import all your Pocket bookmarks (via HTML export). Just go to https://getpocket.com/export, then export the HTML file. An HTML file will be downloaded (like "ril_export.html"). + how_to: Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched. developer: page_title: API clients management welcome_message: Welcome to the wallabag API From 29162bde9dd9f6942b4ee719e5fd3475cf649b30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20L=C5=93uillet?= Date: Sun, 25 May 2025 08:11:54 +0200 Subject: [PATCH 02/11] Pin j0k3r/php-readability to 1.2.10 We pin php-readability to 1.2.10 because of a regression in 1.2.12 Workaround for #8151 Signed-off-by: Kevin Decherf --- composer.json | 4 ++++ composer.lock | 16 ++++++++-------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/composer.json b/composer.json index 2d05b6065..6f2036df8 100644 --- a/composer.json +++ b/composer.json @@ -37,6 +37,9 @@ "email": "hello@wallabag.org", "issues": "https://github.com/wallabag/wallabag/issues" }, + "_comment": [ + "j0k3r/php-readability is pinned to 1.2.10 due to a regression in 1.2.12" + ], "require": { "php": ">=7.4", "ext-ctype": "*", @@ -80,6 +83,7 @@ "html2text/html2text": "^4.3.1", "incenteev/composer-parameter-handler": "^2.1.5", "j0k3r/graby": "^2.4.5", + "j0k3r/php-readability": "1.2.10", "javibravo/simpleue": "^2.1", "jms/serializer": "^3.29.1", "jms/serializer-bundle": "^5.3.1", diff --git a/composer.lock b/composer.lock index a289b813e..2d689bf20 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "313f6b39034c27c58a102bfadb5ae30f", + "content-hash": "931e1540547040b66585959dcbbd4e30", "packages": [ { "name": "babdev/pagerfanta-bundle", @@ -4719,16 +4719,16 @@ }, { "name": "j0k3r/php-readability", - "version": "1.2.12", + "version": "1.2.10", "source": { "type": "git", "url": "https://github.com/j0k3r/php-readability.git", - "reference": "109a22662de0d703f01387e5714ad4f9a03b95c0" + "reference": "563835730692eb369a73fe4bb9a1b44a603c4ce7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/j0k3r/php-readability/zipball/109a22662de0d703f01387e5714ad4f9a03b95c0", - "reference": "109a22662de0d703f01387e5714ad4f9a03b95c0", + "url": "https://api.github.com/repos/j0k3r/php-readability/zipball/563835730692eb369a73fe4bb9a1b44a603c4ce7", + "reference": "563835730692eb369a73fe4bb9a1b44a603c4ce7", "shasum": "" }, "require": { @@ -4740,7 +4740,7 @@ "require-dev": { "friendsofphp/php-cs-fixer": "^2.14", "monolog/monolog": "^1.24|^2.1", - "symfony/phpunit-bridge": "^4.4|^5.3|^6.0|^7.0" + "symfony/phpunit-bridge": "^4.4|^5.3" }, "suggest": { "ext-tidy": "Used to clean up given HTML and to avoid problems with bad HTML structure." @@ -4790,7 +4790,7 @@ ], "support": { "issues": "https://github.com/j0k3r/php-readability/issues", - "source": "https://github.com/j0k3r/php-readability/tree/1.2.12" + "source": "https://github.com/j0k3r/php-readability/tree/1.2.10" }, "funding": [ { @@ -4798,7 +4798,7 @@ "type": "github" } ], - "time": "2025-03-04T09:20:40+00:00" + "time": "2022-06-13T04:15:24+00:00" }, { "name": "javibravo/simpleue", From f82c87b5201275010ac8dfa0222fb1d3eb1a746f Mon Sep 17 00:00:00 2001 From: Kevin Decherf Date: Mon, 2 Jun 2025 13:42:44 +0200 Subject: [PATCH 03/11] import: fix some tests Signed-off-by: Kevin Decherf --- .../ImportBundle/Controller/PocketHtmlControllerTest.php | 8 ++++---- .../ImportBundle/Controller/ShaarliControllerTest.php | 8 ++++---- tests/Wallabag/ImportBundle/fixtures/ril_export.html | 2 +- .../Wallabag/ImportBundle/fixtures/shaarli-bookmarks.html | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/Wallabag/ImportBundle/Controller/PocketHtmlControllerTest.php b/tests/Wallabag/ImportBundle/Controller/PocketHtmlControllerTest.php index 7ae4a247f..b55f8125b 100644 --- a/tests/Wallabag/ImportBundle/Controller/PocketHtmlControllerTest.php +++ b/tests/Wallabag/ImportBundle/Controller/PocketHtmlControllerTest.php @@ -132,14 +132,14 @@ class PocketHtmlControllerTest extends WallabagCoreTestCase ->get(EntityManagerInterface::class) ->getRepository(Entry::class) ->findByUrlAndUserId( - 'https://www.lemonde.fr/disparitions/article/2018/07/05/le-journaliste-et-cineaste-claude-lanzmann-est-mort_5326313_3382.html', + 'https://www.20minutes.fr/paris/4100740-20240715-jo-paris-2024-courir-capitale-maintenant-quais-fermes', $this->getLoggedInUserId() ); $this->assertInstanceOf(Entry::class, $content); - $this->assertNotEmpty($content->getMimetype(), 'Mimetype for https://www.lemonde.fr is ok'); - $this->assertNotEmpty($content->getPreviewPicture(), 'Preview picture for https://www.lemonde.fr is ok'); - $this->assertNotEmpty($content->getLanguage(), 'Language for https://www.lemonde.fr is ok'); + $this->assertNotEmpty($content->getMimetype(), 'Mimetype for 20minutes.fr is ok'); + $this->assertNotEmpty($content->getPreviewPicture(), 'Preview picture for 20minutes.fr is ok'); + $this->assertNotEmpty($content->getLanguage(), 'Language for 20minutes.fr is ok'); } public function testImportWallabagWithEmptyFile() diff --git a/tests/Wallabag/ImportBundle/Controller/ShaarliControllerTest.php b/tests/Wallabag/ImportBundle/Controller/ShaarliControllerTest.php index 8bc9ffb9d..d91680158 100644 --- a/tests/Wallabag/ImportBundle/Controller/ShaarliControllerTest.php +++ b/tests/Wallabag/ImportBundle/Controller/ShaarliControllerTest.php @@ -132,14 +132,14 @@ class ShaarliControllerTest extends WallabagCoreTestCase ->get(EntityManagerInterface::class) ->getRepository(Entry::class) ->findByUrlAndUserId( - 'https://www.lemonde.fr/disparitions/article/2018/07/05/le-journaliste-et-cineaste-claude-lanzmann-est-mort_5326313_3382.html', + 'https://www.20minutes.fr/paris/4100740-20240715-jo-paris-2024-courir-capitale-maintenant-quais-fermes', $this->getLoggedInUserId() ); $this->assertInstanceOf(Entry::class, $content); - $this->assertNotEmpty($content->getMimetype(), 'Mimetype for https://www.lemonde.fr is ok'); - $this->assertNotEmpty($content->getPreviewPicture(), 'Preview picture for https://www.lemonde.fr is ok'); - $this->assertNotEmpty($content->getLanguage(), 'Language for https://www.lemonde.fr is ok'); + $this->assertNotEmpty($content->getMimetype(), 'Mimetype for 20minutes.fr is ok'); + $this->assertNotEmpty($content->getPreviewPicture(), 'Preview picture for 20minutes.fr is ok'); + $this->assertNotEmpty($content->getLanguage(), 'Language for 20minutes.fr is ok'); } public function testImportWallabagWithEmptyFile() diff --git a/tests/Wallabag/ImportBundle/fixtures/ril_export.html b/tests/Wallabag/ImportBundle/fixtures/ril_export.html index 310c092dc..73ddfc0bd 100644 --- a/tests/Wallabag/ImportBundle/fixtures/ril_export.html +++ b/tests/Wallabag/ImportBundle/fixtures/ril_export.html @@ -9,7 +9,7 @@

Unread

diff --git a/tests/Wallabag/ImportBundle/fixtures/shaarli-bookmarks.html b/tests/Wallabag/ImportBundle/fixtures/shaarli-bookmarks.html index ad401ea74..073fdd470 100644 --- a/tests/Wallabag/ImportBundle/fixtures/shaarli-bookmarks.html +++ b/tests/Wallabag/ImportBundle/fixtures/shaarli-bookmarks.html @@ -8,6 +8,6 @@

The Legacy of Firefox OS. In the two years or so since Mozilla… | by Ben Francis | Medium
In the two years or so since Mozilla announced the end of Firefox OS as a Mozilla-run project, the B2G source code has found its way into a surprising number of commercial products. -
Template Filters — Eleventy +
JO Paris 2024 : Où courir dans la capitale maintenant que les quais sont fermés ?

From c1397f43ac0dd957dff9c8b95fc415fc763a01b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20L=C5=93uillet?= Date: Sat, 24 May 2025 17:51:45 +0200 Subject: [PATCH 04/11] Add Pocket CSV import --- app/config/config.yml | 14 + app/config/services.yml | 9 + app/config/services_rabbit.yml | 6 + app/config/services_redis.yml | 16 ++ .../ImportBundle/Command/ImportCommand.php | 11 +- .../Consumer/RabbitMQConsumerTotalProxy.php | 8 +- .../Controller/ImportController.php | 2 + .../Controller/PocketCsvController.php | 57 ++++ .../ImportBundle/Import/PocketCsvImport.php | 156 +++++++++++ .../Resources/views/PocketCsv/index.html.twig | 45 ++++ .../Controller/ImportControllerTest.php | 2 +- .../Controller/PocketCsvControllerTest.php | 155 +++++++++++ .../Import/PocketCsvImportTest.php | 252 ++++++++++++++++++ .../Wallabag/ImportBundle/fixtures/pocket.csv | 8 + tests/Wallabag/ImportBundle/fixtures/test.csv | 0 translations/messages.en.yml | 4 + translations/messages.fr.yml | 13 + 17 files changed, 754 insertions(+), 4 deletions(-) create mode 100644 src/Wallabag/ImportBundle/Controller/PocketCsvController.php create mode 100644 src/Wallabag/ImportBundle/Import/PocketCsvImport.php create mode 100644 src/Wallabag/ImportBundle/Resources/views/PocketCsv/index.html.twig create mode 100644 tests/Wallabag/ImportBundle/Controller/PocketCsvControllerTest.php create mode 100644 tests/Wallabag/ImportBundle/Import/PocketCsvImportTest.php create mode 100644 tests/Wallabag/ImportBundle/fixtures/pocket.csv create mode 100644 tests/Wallabag/ImportBundle/fixtures/test.csv diff --git a/app/config/config.yml b/app/config/config.yml index 1a522ddbe..c4c151200 100644 --- a/app/config/config.yml +++ b/app/config/config.yml @@ -293,6 +293,11 @@ old_sound_rabbit_mq: exchange_options: name: 'wallabag.import.pocket_html' type: topic + import_pocket_csv: + connection: default + exchange_options: + name: 'wallabag.import.pocket_csv' + type: topic consumers: import_pocket: connection: default @@ -411,6 +416,15 @@ old_sound_rabbit_mq: name: 'wallabag.import.pocket_html' callback: wallabag_import.consumer.amqp.pocket_html qos_options: {prefetch_count: "%rabbitmq_prefetch_count%"} + import_pocket_csv: + connection: default + exchange_options: + name: 'wallabag.import.pocket_csv' + type: topic + queue_options: + name: 'wallabag.import.pocket_csv' + callback: wallabag_import.consumer.amqp.pocket_csv + qos_options: {prefetch_count: "%rabbitmq_prefetch_count%"} fos_js_routing: routes_to_expose: diff --git a/app/config/services.yml b/app/config/services.yml index cc7c5329c..311e7e72d 100644 --- a/app/config/services.yml +++ b/app/config/services.yml @@ -131,6 +131,11 @@ services: $rabbitMqProducer: '@old_sound_rabbit_mq.import_pocket_html_producer' $redisProducer: '@wallabag_import.producer.redis.pocket_html' + Wallabag\ImportBundle\Controller\PocketCsvController: + arguments: + $rabbitMqProducer: '@old_sound_rabbit_mq.import_pocket_csv_producer' + $redisProducer: '@wallabag_import.producer.redis.pocket_csv' + Wallabag\ImportBundle\: resource: '../../src/Wallabag/ImportBundle/*' exclude: '../../src/Wallabag/ImportBundle/{Consumer,Controller,Redis}' @@ -412,6 +417,10 @@ services: tags: - { name: wallabag_import.import, alias: pocket_html } + Wallabag\ImportBundle\Import\PocketCsvImport: + tags: + - { name: wallabag_import.import, alias: pocket_csv } + # to factorize the proximity and bypass translation for prev & next pagerfanta.view.default_wallabag: class: Pagerfanta\View\OptionableView diff --git a/app/config/services_rabbit.yml b/app/config/services_rabbit.yml index e6963733a..fa182d124 100644 --- a/app/config/services_rabbit.yml +++ b/app/config/services_rabbit.yml @@ -21,6 +21,7 @@ services: $omnivoreConsumer: '@old_sound_rabbit_mq.import_omnivore_consumer' $shaarliConsumer: '@old_sound_rabbit_mq.import_shaarli_consumer' $pocketHtmlConsumer: '@old_sound_rabbit_mq.import_pocket_html_consumer' + $pocketCsvConsumer: '@old_sound_rabbit_mq.import_pocket_csv_consumer' wallabag_import.consumer.amqp.pocket: class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer @@ -86,3 +87,8 @@ services: class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer arguments: $import: '@Wallabag\ImportBundle\Import\PocketHtmlImport' + + wallabag_import.consumer.amqp.pocket_csv: + class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer + arguments: + $import: '@Wallabag\ImportBundle\Import\PocketCsvImport' diff --git a/app/config/services_redis.yml b/app/config/services_redis.yml index baeb1ccef..5280078f1 100644 --- a/app/config/services_redis.yml +++ b/app/config/services_redis.yml @@ -212,3 +212,19 @@ services: class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer arguments: $import: '@Wallabag\ImportBundle\Import\PocketHtmlImport' + + # pocket csv + wallabag_import.queue.redis.pocket_csv: + class: Simpleue\Queue\RedisQueue + arguments: + $queueName: "wallabag.import.pocket_csv" + + wallabag_import.producer.redis.pocket_csv: + class: Wallabag\ImportBundle\Redis\Producer + arguments: + - "@wallabag_import.queue.redis.pocket_csv" + + wallabag_import.consumer.redis.pocket_csv: + class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer + arguments: + $import: '@Wallabag\ImportBundle\Import\PocketCsvImport' diff --git a/src/Wallabag/ImportBundle/Command/ImportCommand.php b/src/Wallabag/ImportBundle/Command/ImportCommand.php index 76bd4f451..2d9f876db 100644 --- a/src/Wallabag/ImportBundle/Command/ImportCommand.php +++ b/src/Wallabag/ImportBundle/Command/ImportCommand.php @@ -18,6 +18,7 @@ use Wallabag\ImportBundle\Import\FirefoxImport; use Wallabag\ImportBundle\Import\InstapaperImport; use Wallabag\ImportBundle\Import\OmnivoreImport; use Wallabag\ImportBundle\Import\PinboardImport; +use Wallabag\ImportBundle\Import\PocketCsvImport; use Wallabag\ImportBundle\Import\PocketHtmlImport; use Wallabag\ImportBundle\Import\ReadabilityImport; use Wallabag\ImportBundle\Import\ShaarliImport; @@ -43,6 +44,7 @@ class ImportCommand extends Command private ElcuratorImport $elcuratorImport; private ShaarliImport $shaarliImport; private PocketHtmlImport $pocketHtmlImport; + private PocketCsvImport $pocketCsvImport; public function __construct( EntityManagerInterface $entityManager, @@ -59,7 +61,8 @@ class ImportCommand extends Command ElcuratorImport $elcuratorImport, OmnivoreImport $omnivoreImport, ShaarliImport $shaarliImport, - PocketHtmlImport $pocketHtmlImport + PocketHtmlImport $pocketHtmlImport, + PocketCsvImport $pocketCsvImport ) { $this->entityManager = $entityManager; $this->tokenStorage = $tokenStorage; @@ -76,6 +79,7 @@ class ImportCommand extends Command $this->elcuratorImport = $elcuratorImport; $this->shaarliImport = $shaarliImport; $this->pocketHtmlImport = $pocketHtmlImport; + $this->pocketCsvImport = $pocketCsvImport; parent::__construct(); } @@ -87,7 +91,7 @@ class ImportCommand extends Command ->setDescription('Import entries from a JSON export') ->addArgument('username', InputArgument::REQUIRED, 'User to populate') ->addArgument('filepath', InputArgument::REQUIRED, 'Path to the JSON file') - ->addOption('importer', null, InputOption::VALUE_OPTIONAL, 'The importer to use: v1, v2, instapaper, pinboard, delicious, readability, firefox, chrome, elcurator, shaarli or pocket', 'v1') + ->addOption('importer', null, InputOption::VALUE_OPTIONAL, 'The importer to use: v1, v2, instapaper, pinboard, delicious, readability, firefox, chrome, elcurator, shaarli, pocket or pocket_csv', 'v1') ->addOption('markAsRead', null, InputOption::VALUE_OPTIONAL, 'Mark all entries as read', false) ->addOption('useUserId', null, InputOption::VALUE_NONE, 'Use user id instead of username to find account') ->addOption('disableContentUpdate', null, InputOption::VALUE_NONE, 'Disable fetching updated content from URL') @@ -159,6 +163,9 @@ class ImportCommand extends Command case 'pocket': $import = $this->pocketHtmlImport; break; + case 'pocket_csv': + $import = $this->pocketCsvImport; + break; default: $import = $this->wallabagV1Import; } diff --git a/src/Wallabag/ImportBundle/Consumer/RabbitMQConsumerTotalProxy.php b/src/Wallabag/ImportBundle/Consumer/RabbitMQConsumerTotalProxy.php index 0d531d884..eaba83592 100644 --- a/src/Wallabag/ImportBundle/Consumer/RabbitMQConsumerTotalProxy.php +++ b/src/Wallabag/ImportBundle/Consumer/RabbitMQConsumerTotalProxy.php @@ -23,6 +23,7 @@ class RabbitMQConsumerTotalProxy private Consumer $omnivoreConsumer; private Consumer $shaarliConsumer; private Consumer $pocketHtmlConsumer; + private Consumer $pocketCsvConsumer; public function __construct( Consumer $pocketConsumer, @@ -37,7 +38,8 @@ class RabbitMQConsumerTotalProxy Consumer $elcuratorConsumer, Consumer $omnivoreConsumer, Consumer $shaarliConsumer, - Consumer $pocketHtmlConsumer + Consumer $pocketHtmlConsumer, + Consumer $pocketCsvConsumer ) { $this->pocketConsumer = $pocketConsumer; $this->readabilityConsumer = $readabilityConsumer; @@ -52,6 +54,7 @@ class RabbitMQConsumerTotalProxy $this->omnivoreConsumer = $omnivoreConsumer; $this->shaarliConsumer = $shaarliConsumer; $this->pocketHtmlConsumer = $pocketHtmlConsumer; + $this->pocketCsvConsumer = $pocketCsvConsumer; } /** @@ -105,6 +108,9 @@ class RabbitMQConsumerTotalProxy case 'pocket_html': $consumer = $this->pocketHtmlConsumer; break; + case 'pocket_csv': + $consumer = $this->pocketCsvConsumer; + break; default: return 0; } diff --git a/src/Wallabag/ImportBundle/Controller/ImportController.php b/src/Wallabag/ImportBundle/Controller/ImportController.php index 6e6e5d7c1..a155e6eb5 100644 --- a/src/Wallabag/ImportBundle/Controller/ImportController.php +++ b/src/Wallabag/ImportBundle/Controller/ImportController.php @@ -60,6 +60,7 @@ class ImportController extends AbstractController + $this->rabbitMQConsumerTotalProxy->getTotalMessage('omnivore') + $this->rabbitMQConsumerTotalProxy->getTotalMessage('shaarli') + $this->rabbitMQConsumerTotalProxy->getTotalMessage('pocket_html') + + $this->rabbitMQConsumerTotalProxy->getTotalMessage('pocket_csv') ; } catch (\Exception $e) { $rabbitNotInstalled = true; @@ -81,6 +82,7 @@ class ImportController extends AbstractController + $redis->llen('wallabag.import.omnivore') + $redis->llen('wallabag.import.shaarli') + $redis->llen('wallabag.import.pocket_html') + + $redis->llen('wallabag.import.pocket_csv') ; } catch (\Exception $e) { $redisNotInstalled = true; diff --git a/src/Wallabag/ImportBundle/Controller/PocketCsvController.php b/src/Wallabag/ImportBundle/Controller/PocketCsvController.php new file mode 100644 index 000000000..4eef3c0c8 --- /dev/null +++ b/src/Wallabag/ImportBundle/Controller/PocketCsvController.php @@ -0,0 +1,57 @@ +pocketCsvImport = $pocketCsvImport; + $this->craueConfig = $craueConfig; + $this->rabbitMqProducer = $rabbitMqProducer; + $this->redisProducer = $redisProducer; + } + + /** + * @Route("/pocket_csv", name="import_pocket_csv") + */ + public function indexAction(Request $request, TranslatorInterface $translator) + { + return parent::indexAction($request, $translator); + } + + /** + * {@inheritdoc} + */ + protected function getImportService() + { + if ($this->craueConfig->get('import_with_rabbitmq')) { + $this->pocketCsvImport->setProducer($this->rabbitMqProducer); + } elseif ($this->craueConfig->get('import_with_redis')) { + $this->pocketCsvImport->setProducer($this->redisProducer); + } + + return $this->pocketCsvImport; + } + + /** + * {@inheritdoc} + */ + protected function getImportTemplate() + { + return '@WallabagImport/PocketCsv/index.html.twig'; + } +} diff --git a/src/Wallabag/ImportBundle/Import/PocketCsvImport.php b/src/Wallabag/ImportBundle/Import/PocketCsvImport.php new file mode 100644 index 000000000..62b830b0f --- /dev/null +++ b/src/Wallabag/ImportBundle/Import/PocketCsvImport.php @@ -0,0 +1,156 @@ +filepath = $filepath; + + return $this; + } + + /** + * {@inheritdoc} + */ + public function validateEntry(array $importedEntry) + { + if (empty($importedEntry['url'])) { + return false; + } + + return true; + } + + public function import() + { + if (!$this->user) { + $this->logger->error('Pocket CSV Import: user is not defined'); + + return false; + } + + if (!file_exists($this->filepath) || !is_readable($this->filepath)) { + $this->logger->error('Pocket CSV Import: unable to read file', ['filepath' => $this->filepath]); + + return false; + } + + $entries = []; + $handle = fopen($this->filepath, 'r'); + while (false !== ($data = fgetcsv($handle, 10240))) { + if ('title' === $data[0]) { + continue; + } + + $entries[] = [ + 'url' => $data[1], + 'title' => $data[0], + 'is_archived' => 'archive' === $data[4], + 'created_at' => $data[2], + 'tags' => $data[3], + ]; + } + fclose($handle); + + if (empty($entries)) { + $this->logger->error('PocketCsvImport: no entries in imported file'); + + return false; + } + + if ($this->producer) { + $this->parseEntriesForProducer($entries); + + return true; + } + + $this->parseEntries($entries); + + return true; + } + + /** + * {@inheritdoc} + */ + public function parseEntry(array $importedEntry) + { + $existingEntry = $this->em + ->getRepository(Entry::class) + ->findByUrlAndUserId($importedEntry['url'], $this->user->getId()); + + if (false !== $existingEntry) { + ++$this->skippedEntries; + + return; + } + + $entry = new Entry($this->user); + $entry->setUrl($importedEntry['url']); + $entry->setTitle($importedEntry['title']); + + // update entry with content (in case fetching failed, the given entry will be return) + $this->fetchContent($entry, $importedEntry['url'], $importedEntry); + + if (!empty($importedEntry['tags'])) { + $tags = str_replace('|', ',', $importedEntry['tags']); + $this->tagsAssigner->assignTagsToEntry( + $entry, + $tags, + $this->em->getUnitOfWork()->getScheduledEntityInsertions() + ); + } + + $entry->updateArchived($importedEntry['is_archived']); + $entry->setCreatedAt(\DateTime::createFromFormat('U', $importedEntry['created_at'])); + + $this->em->persist($entry); + ++$this->importedEntries; + + return $entry; + } + + /** + * {@inheritdoc} + */ + protected function setEntryAsRead(array $importedEntry) + { + $importedEntry['is_archived'] = 'archive'; + + return $importedEntry; + } +} diff --git a/src/Wallabag/ImportBundle/Resources/views/PocketCsv/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/PocketCsv/index.html.twig new file mode 100644 index 000000000..229171a8d --- /dev/null +++ b/src/Wallabag/ImportBundle/Resources/views/PocketCsv/index.html.twig @@ -0,0 +1,45 @@ +{% extends "@WallabagCore/layout.html.twig" %} + +{% block title %}{{ 'import.pocket_csv.page_title'|trans }}{% endblock %} + +{% block content %} +

+
+
+ {% include '@WallabagImport/Import/_information.html.twig' %} + +
+
{{ import.description|trans|raw }}
+

{{ 'import.pocket_csv.how_to'|trans }}

+ +
+ {{ form_start(form, {'method': 'POST'}) }} + {{ form_errors(form) }} +
+
+ {{ form_errors(form.file) }} +
+ {{ form.file.vars.label|trans }} + {{ form_widget(form.file) }} +
+
+ +
+
+
+
{{ 'import.form.mark_as_read_title'|trans }}
+ {{ form_widget(form.mark_as_read) }} + {{ form_label(form.mark_as_read) }} +
+
+ + {{ form_widget(form.save, {'attr': {'class': 'btn waves-effect waves-light'}}) }} + + {{ form_rest(form) }} + +
+
+
+
+
+{% endblock %} diff --git a/tests/Wallabag/ImportBundle/Controller/ImportControllerTest.php b/tests/Wallabag/ImportBundle/Controller/ImportControllerTest.php index 76b802b6e..59dcbb7de 100644 --- a/tests/Wallabag/ImportBundle/Controller/ImportControllerTest.php +++ b/tests/Wallabag/ImportBundle/Controller/ImportControllerTest.php @@ -24,6 +24,6 @@ class ImportControllerTest extends WallabagCoreTestCase $crawler = $client->request('GET', '/import/'); $this->assertSame(200, $client->getResponse()->getStatusCode()); - $this->assertSame(13, $crawler->filter('blockquote')->count()); + $this->assertSame(14, $crawler->filter('blockquote')->count()); } } diff --git a/tests/Wallabag/ImportBundle/Controller/PocketCsvControllerTest.php b/tests/Wallabag/ImportBundle/Controller/PocketCsvControllerTest.php new file mode 100644 index 000000000..ae8c08d25 --- /dev/null +++ b/tests/Wallabag/ImportBundle/Controller/PocketCsvControllerTest.php @@ -0,0 +1,155 @@ +logInAs('admin'); + $client = $this->getTestClient(); + + $crawler = $client->request('GET', '/import/pocket_csv'); + + $this->assertSame(200, $client->getResponse()->getStatusCode()); + $this->assertSame(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); + $this->assertSame(1, $crawler->filter('input[type=file]')->count()); + } + + public function testImportPocketCsvWithRabbitEnabled() + { + $this->logInAs('admin'); + $client = $this->getTestClient(); + + $client->getContainer()->get(Config::class)->set('import_with_rabbitmq', 1); + + $crawler = $client->request('GET', '/import/pocket_csv'); + + $this->assertSame(200, $client->getResponse()->getStatusCode()); + $this->assertSame(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); + $this->assertSame(1, $crawler->filter('input[type=file]')->count()); + + $client->getContainer()->get(Config::class)->set('import_with_rabbitmq', 0); + } + + public function testImportPocketCsvBadFile() + { + $this->logInAs('admin'); + $client = $this->getTestClient(); + + $crawler = $client->request('GET', '/import/pocket_csv'); + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $data = [ + 'upload_import_file[file]' => '', + ]; + + $client->submit($form, $data); + + $this->assertSame(200, $client->getResponse()->getStatusCode()); + } + + public function testImportPocketCsvWithRedisEnabled() + { + $this->checkRedis(); + $this->logInAs('admin'); + $client = $this->getTestClient(); + $client->getContainer()->get(Config::class)->set('import_with_redis', 1); + + $crawler = $client->request('GET', '/import/pocket_csv'); + + $this->assertSame(200, $client->getResponse()->getStatusCode()); + $this->assertSame(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); + $this->assertSame(1, $crawler->filter('input[type=file]')->count()); + + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $file = new UploadedFile(__DIR__ . '/../fixtures/pocket.csv', 'Bookmarks'); + + $data = [ + 'upload_import_file[file]' => $file, + ]; + + $client->submit($form, $data); + + $this->assertSame(302, $client->getResponse()->getStatusCode()); + + $crawler = $client->followRedirect(); + + $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text'])); + $this->assertStringContainsString('flashes.import.notice.summary', $body[0]); + + $this->assertNotEmpty($client->getContainer()->get(Client::class)->lpop('wallabag.import.pocket_csv')); + + $client->getContainer()->get(Config::class)->set('import_with_redis', 0); + } + + public function testImportWallabagWithPocketCsvFile() + { + $this->logInAs('admin'); + $client = $this->getTestClient(); + + $crawler = $client->request('GET', '/import/pocket_csv'); + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $file = new UploadedFile(__DIR__ . '/../fixtures/pocket.csv', 'Bookmarks'); + + $data = [ + 'upload_import_file[file]' => $file, + ]; + + $client->submit($form, $data); + + $this->assertSame(302, $client->getResponse()->getStatusCode()); + + $crawler = $client->followRedirect(); + + $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text'])); + $this->assertStringContainsString('flashes.import.notice.summary', $body[0]); + + $content = $client->getContainer() + ->get(EntityManagerInterface::class) + ->getRepository(Entry::class) + ->findByUrlAndUserId( + 'https://jp-lambert.me/est-ce-que-jai-besoin-d-un-scrum-master-604f5a471c73', + $this->getLoggedInUserId() + ); + + $this->assertInstanceOf(Entry::class, $content); + $this->assertNotEmpty($content->getMimetype(), 'Mimetype for jp-lambert.me is ok'); + $this->assertNotEmpty($content->getPreviewPicture(), 'Preview picture for jp-lambert.me is ok'); + $this->assertNotEmpty($content->getLanguage(), 'Language for jp-lambert.me is ok'); + $this->assertCount(1, $content->getTags()); + } + + public function testImportWallabagWithEmptyFile() + { + $this->logInAs('admin'); + $client = $this->getTestClient(); + + $crawler = $client->request('GET', '/import/pocket_csv'); + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $file = new UploadedFile(__DIR__ . '/../fixtures/test.csv', 'test.csv'); + + $data = [ + 'upload_import_file[file]' => $file, + ]; + + $client->submit($form, $data); + + $this->assertSame(302, $client->getResponse()->getStatusCode()); + + $crawler = $client->followRedirect(); + + $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text'])); + $this->assertStringContainsString('flashes.import.notice.failed', $body[0]); + } +} diff --git a/tests/Wallabag/ImportBundle/Import/PocketCsvImportTest.php b/tests/Wallabag/ImportBundle/Import/PocketCsvImportTest.php new file mode 100644 index 000000000..c3a7024b6 --- /dev/null +++ b/tests/Wallabag/ImportBundle/Import/PocketCsvImportTest.php @@ -0,0 +1,252 @@ +getPocketCsvImport(); + + $this->assertSame('Pocket CSV', $pocketCsvImport->getName()); + $this->assertNotEmpty($pocketCsvImport->getUrl()); + $this->assertSame('import.pocket_csv.description', $pocketCsvImport->getDescription()); + } + + public function testImport() + { + $pocketCsvImport = $this->getPocketCsvImport(false, 7); + $pocketCsvImport->setFilepath(__DIR__ . '/../fixtures/pocket.csv'); + + $entryRepo = $this->getMockBuilder(EntryRepository::class) + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->exactly(7)) + ->method('findByUrlAndUserId') + ->willReturn(false); + + $this->em + ->expects($this->any()) + ->method('getRepository') + ->willReturn($entryRepo); + + $entry = $this->getMockBuilder(Entry::class) + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy + ->expects($this->exactly(7)) + ->method('updateEntry') + ->willReturn($entry); + + $res = $pocketCsvImport->import(); + + $this->assertTrue($res); + $this->assertSame(['skipped' => 0, 'imported' => 7, 'queued' => 0], $pocketCsvImport->getSummary()); + } + + public function testImportAndMarkAllAsRead() + { + $pocketCsvImport = $this->getPocketCsvImport(false, 1); + $pocketCsvImport->setFilepath(__DIR__ . '/../fixtures/pocket.csv'); + + $entryRepo = $this->getMockBuilder(EntryRepository::class) + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->exactly(7)) + ->method('findByUrlAndUserId') + ->will($this->onConsecutiveCalls(false, true)); + + $this->em + ->expects($this->any()) + ->method('getRepository') + ->willReturn($entryRepo); + + $this->contentProxy + ->expects($this->exactly(1)) + ->method('updateEntry') + ->willReturn(new Entry($this->user)); + + // check that every entry persisted are archived + $this->em + ->expects($this->any()) + ->method('persist') + ->with($this->callback(fn ($persistedEntry) => (bool) $persistedEntry->isArchived())); + + $res = $pocketCsvImport + ->setMarkAsRead(true) + ->import(); + + $this->assertTrue($res); + + $this->assertSame(['skipped' => 6, 'imported' => 1, 'queued' => 0], $pocketCsvImport->getSummary()); + } + + public function testImportWithRabbit() + { + $pocketCsvImport = $this->getPocketCsvImport(); + $pocketCsvImport->setFilepath(__DIR__ . '/../fixtures/pocket.csv'); + + $entryRepo = $this->getMockBuilder(EntryRepository::class) + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->never()) + ->method('findByUrlAndUserId'); + + $this->em + ->expects($this->never()) + ->method('getRepository'); + + $entry = $this->getMockBuilder(Entry::class) + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy + ->expects($this->never()) + ->method('updateEntry'); + + $producer = $this->getMockBuilder(\OldSound\RabbitMqBundle\RabbitMq\Producer::class) + ->disableOriginalConstructor() + ->getMock(); + + $producer + ->expects($this->exactly(7)) + ->method('publish'); + + $pocketCsvImport->setProducer($producer); + + $res = $pocketCsvImport->setMarkAsRead(true)->import(); + + $this->assertTrue($res); + $this->assertSame(['skipped' => 0, 'imported' => 0, 'queued' => 7], $pocketCsvImport->getSummary()); + } + + public function testImportWithRedis() + { + $pocketCsvImport = $this->getPocketCsvImport(); + $pocketCsvImport->setFilepath(__DIR__ . '/../fixtures/pocket.csv'); + + $entryRepo = $this->getMockBuilder(EntryRepository::class) + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->never()) + ->method('findByUrlAndUserId'); + + $this->em + ->expects($this->never()) + ->method('getRepository'); + + $entry = $this->getMockBuilder(Entry::class) + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy + ->expects($this->never()) + ->method('updateEntry'); + + $factory = new RedisMockFactory(); + $redisMock = $factory->getAdapter(Client::class, true); + + $queue = new RedisQueue($redisMock, 'pocket_csv'); + $producer = new Producer($queue); + + $pocketCsvImport->setProducer($producer); + + $res = $pocketCsvImport->setMarkAsRead(true)->import(); + + $this->assertTrue($res); + $this->assertSame(['skipped' => 0, 'imported' => 0, 'queued' => 7], $pocketCsvImport->getSummary()); + + $this->assertNotEmpty($redisMock->lpop('pocket_csv')); + } + + public function testImportBadFile() + { + $pocketCsvImport = $this->getPocketCsvImport(); + $pocketCsvImport->setFilepath(__DIR__ . '/../fixtures/Import/wallabag-v1.jsonx'); + + $res = $pocketCsvImport->import(); + + $this->assertFalse($res); + + $records = $this->logHandler->getRecords(); + $this->assertStringContainsString('Pocket CSV Import: unable to read file', $records[0]['message']); + $this->assertSame('ERROR', $records[0]['level_name']); + } + + public function testImportUserNotDefined() + { + $pocketCsvImport = $this->getPocketCsvImport(true); + $pocketCsvImport->setFilepath(__DIR__ . '/../fixtures/pocket.csv'); + + $res = $pocketCsvImport->import(); + + $this->assertFalse($res); + + $records = $this->logHandler->getRecords(); + $this->assertStringContainsString('Pocket CSV Import: user is not defined', $records[0]['message']); + $this->assertSame('ERROR', $records[0]['level_name']); + } + + private function getPocketCsvImport($unsetUser = false, $dispatched = 0) + { + $this->user = new User(); + + $this->em = $this->getMockBuilder(EntityManager::class) + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy = $this->getMockBuilder(ContentProxy::class) + ->disableOriginalConstructor() + ->getMock(); + + $this->tagsAssigner = $this->getMockBuilder(TagsAssigner::class) + ->disableOriginalConstructor() + ->getMock(); + + $dispatcher = $this->getMockBuilder(EventDispatcher::class) + ->disableOriginalConstructor() + ->getMock(); + + $dispatcher + ->expects($this->exactly($dispatched)) + ->method('dispatch'); + + $this->logHandler = new TestHandler(); + $logger = new Logger('test', [$this->logHandler]); + + $wallabag = new PocketCsvImport($this->em, $this->contentProxy, $this->tagsAssigner, $dispatcher, $logger); + + if (false === $unsetUser) { + $wallabag->setUser($this->user); + } + + return $wallabag; + } +} diff --git a/tests/Wallabag/ImportBundle/fixtures/pocket.csv b/tests/Wallabag/ImportBundle/fixtures/pocket.csv new file mode 100644 index 000000000..e31516feb --- /dev/null +++ b/tests/Wallabag/ImportBundle/fixtures/pocket.csv @@ -0,0 +1,8 @@ +title,url,time_added,tags,status +You Might Not Need jQuery,http://youmightnotneedjquery.com/,1600322788,,unread +Est-ce que j’ai besoin d’un Scrum Master ? | by Jean-Pierre Lambert | Jean-,https://jp-lambert.me/est-ce-que-jai-besoin-d-un-scrum-master-604f5a471c73,1600172739,,unread +"Avec les accusés d’El Halia, par Gisèle Halimi (Le Monde diplomatique, sept",https://www.monde-diplomatique.fr/2020/09/HALIMI/62165,1599806347,,unread +ArchiveBox question: How do I import links from a RSS feed?,https://www.reddit.com/r/DataHoarder/comments/ioupbk/archivebox_question_how_do_i_import_links_from_a/,1600961496,,archive +« Tu vas pleurer les premières fois » : que se passe-t-il au sein du studio,https://www.numerama.com/politique/646826-tu-vas-pleurer-les-premieres-fois-que-se-passe-t-il-au-sein-du-studio-dubisoft-derriere-trackmania.html#utm_medium=distibuted&utm_source=rss&utm_campaign=646826,1599809025,,unread +Comment Konbini s’est fait piéger par un « père masculiniste »,https://www.nouvelobs.com/rue89/20200911.OBS33165/comment-konbini-s-est-fait-pieger-par-un-pere-masculiniste.html,1599819251,,archive +Des abeilles pour résoudre les « conflits » entre les humains et les élépha,https://reporterre.net/Des-abeilles-pour-resoudre-les-conflits-entre-les-humains-et-les-elephants,1599890673,,unread diff --git a/tests/Wallabag/ImportBundle/fixtures/test.csv b/tests/Wallabag/ImportBundle/fixtures/test.csv new file mode 100644 index 000000000..e69de29bb diff --git a/translations/messages.en.yml b/translations/messages.en.yml index e5a195bbb..ed09d6347 100644 --- a/translations/messages.en.yml +++ b/translations/messages.en.yml @@ -542,6 +542,10 @@ import: page_title: Import > Pocket HTML description: This importer will import all your Pocket bookmarks (via HTML export). Just go to https://getpocket.com/export, then export the HTML file. An HTML file will be downloaded (like "ril_export.html"). how_to: Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched. + pocket_csv: + page_title: Import > Pocket CSV + description: This importer will import all your Pocket bookmarks (via CSV export). Just go to https://getpocket.com/export, then export the file. A ZIP file will be downloaded (like "pocket.zip"). Extract it, you will obtain a CSV file, called "part_000000.csv". + how_to: Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched. developer: page_title: API clients management welcome_message: Welcome to the wallabag API diff --git a/translations/messages.fr.yml b/translations/messages.fr.yml index c10a124ff..421e30996 100644 --- a/translations/messages.fr.yml +++ b/translations/messages.fr.yml @@ -530,6 +530,19 @@ import: page_title: Importer > del.icio.us how_to: Choisissez le fichier de votre export Delicious et cliquez sur le bouton ci-dessous pour l'importer. description: Depuis 2021, vous pouvez à nouveau exporter vos données depuis Delicious (https://del.icio.us/export). Choisissez le format "JSON" et téléchargez le (un fichier du genre "delicious_export.2021.02.06_21.10.json"). + shaarli: + page_title: Importer > Shaarli + description: Cet importateur importera toutes vos signets Shaarli. Il suffit d'aller à la section Outils, puis dans « Base de données d'exportation », choisissez vos signets et exportez-les. Vous obtiendrez un fichier HTML. + how_to: Veuillez sélectionner le fichier de sauvegarde de signet et cliquez sur le bouton ci-dessous pour l'importer. Notez que le processus peut prendre beaucoup de temps puisque tous les articles doivent être récupérés. + pocket_html: + page_title: Importer > Pocket HTML + description: Cet importateur importera toutes vos signets Pocket (via exportation HTML). Il suffit d'aller à https://getpocket.com/export, puis d'exporter le fichier HTML. Un fichier HTML sera téléchargé (comme « ril_export.html »). + how_to: Veuillez choisir le fichier de sauvegarde de signets et cliquez sur le bouton ci-dessous pour l'importer. Pensez au fait que le processus peut prendre longtemps puisque tous les articles doivent être récupérés. + pocket_csv: + page_title: Importer > Pocket CSV + description: Cet importateur importera toutes vos signets Pocket (via exportation CSV). Il suffit d'aller à https://getpocket.com/export, puis d'exporter le fichier. Un fichier ZIP sera téléchargé (comme « pocket.zip »). Décompressez le et vous obtiendrez un fichier CSV appelé "part_000000.csv". + how_to: Veuillez choisir le fichier de sauvegarde de signets et cliquez sur le bouton ci-dessous pour l'importer. Pensez au fait que le processus peut prendre longtemps puisque tous les articles doivent être récupérés. + developer: page_title: Gestion des clients API welcome_message: Bienvenue sur l’API de wallabag From 27d66d9e1d82ef7793a5ed1201254a044b49af82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20L=C5=93uillet?= Date: Tue, 27 May 2025 07:55:47 +0200 Subject: [PATCH 05/11] Add 'application/vnd.ms-excel' to allowed MIME types config Backport #8036 --- app/config/wallabag.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/app/config/wallabag.yml b/app/config/wallabag.yml index 2c5e53a00..44cadc57d 100644 --- a/app/config/wallabag.yml +++ b/app/config/wallabag.yml @@ -167,5 +167,11 @@ wallabag_core: rule: _all ~ "https?://www\.lemonde\.fr/tiny.*" wallabag_import: - allow_mimetypes: ['application/octet-stream', 'application/json', 'text/plain', 'text/csv', 'text/html'] + allow_mimetypes: + - 'application/octet-stream' + - 'application/json' + - 'text/plain' + - 'text/csv' + - 'text/html' + - 'application/vnd.ms-excel' resource_dir: "%kernel.project_dir%/web/uploads/import" From c4240c866b547fad4485a234beadc40b8eb93cb5 Mon Sep 17 00:00:00 2001 From: Kevin Decherf Date: Tue, 3 Jun 2025 08:56:10 +0200 Subject: [PATCH 06/11] PocketCsvImport: rework test to ensure we import all entries This also adds a test for #8217 Signed-off-by: Kevin Decherf --- .../Controller/PocketCsvControllerTest.php | 33 +++++++++++++------ .../Wallabag/ImportBundle/fixtures/pocket.csv | 4 ++- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/tests/Wallabag/ImportBundle/Controller/PocketCsvControllerTest.php b/tests/Wallabag/ImportBundle/Controller/PocketCsvControllerTest.php index ae8c08d25..d5a3d4df6 100644 --- a/tests/Wallabag/ImportBundle/Controller/PocketCsvControllerTest.php +++ b/tests/Wallabag/ImportBundle/Controller/PocketCsvControllerTest.php @@ -114,19 +114,32 @@ class PocketCsvControllerTest extends WallabagCoreTestCase $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text'])); $this->assertStringContainsString('flashes.import.notice.summary', $body[0]); - $content = $client->getContainer() + $entries = $client->getContainer() ->get(EntityManagerInterface::class) ->getRepository(Entry::class) - ->findByUrlAndUserId( - 'https://jp-lambert.me/est-ce-que-jai-besoin-d-un-scrum-master-604f5a471c73', - $this->getLoggedInUserId() - ); + ->findBy(['user' => $this->getLoggedInUserId()]); - $this->assertInstanceOf(Entry::class, $content); - $this->assertNotEmpty($content->getMimetype(), 'Mimetype for jp-lambert.me is ok'); - $this->assertNotEmpty($content->getPreviewPicture(), 'Preview picture for jp-lambert.me is ok'); - $this->assertNotEmpty($content->getLanguage(), 'Language for jp-lambert.me is ok'); - $this->assertCount(1, $content->getTags()); + $expectedEntries = [ + 'http://youmightnotneedjquery.com/,1600322788', + 'https://jp-lambert.me/est-ce-que-jai-besoin-d-un-scrum-master-604f5a471c73', + 'https://www.monde-diplomatique.fr/2020/09/HALIMI/62165', + 'https://www.reddit.com/r/DataHoarder/comments/ioupbk/archivebox_question_how_do_i_import_links_from_a/', + 'https://www.numerama.com/politique/646826-tu-vas-pleurer-les-premieres-fois-que-se-passe-t-il-au-sein-du-studio-dubisoft-derriere-trackmania.html#utm_medium=distibuted&utm_source=rss&utm_campaign=646826', + 'https://www.nouvelobs.com/rue89/20200911.OBS33165/comment-konbini-s-est-fait-pieger-par-un-pere-masculiniste.html', + 'https://reporterre.net/Des-abeilles-pour-resoudre-les-conflits-entre-les-humains-et-les-elephants', + ]; + + $matchedEntries = array_map(function ($expectedUrl) use ($entries) { + foreach ($entries as $entry) { + if ($entry->getUrl() === $expectedUrl) { + return $entry; + } + } + + return null; + }, $expectedEntries); + + $this->assertCount(\count($expectedEntries), $matchedEntries, 'Should have 7 entries imported from pocket.csv'); } public function testImportWallabagWithEmptyFile() diff --git a/tests/Wallabag/ImportBundle/fixtures/pocket.csv b/tests/Wallabag/ImportBundle/fixtures/pocket.csv index e31516feb..085d3d86e 100644 --- a/tests/Wallabag/ImportBundle/fixtures/pocket.csv +++ b/tests/Wallabag/ImportBundle/fixtures/pocket.csv @@ -5,4 +5,6 @@ Est-ce que j’ai besoin d’un Scrum Master ? | by Jean-Pierre Lambert | Jean-, ArchiveBox question: How do I import links from a RSS feed?,https://www.reddit.com/r/DataHoarder/comments/ioupbk/archivebox_question_how_do_i_import_links_from_a/,1600961496,,archive « Tu vas pleurer les premières fois » : que se passe-t-il au sein du studio,https://www.numerama.com/politique/646826-tu-vas-pleurer-les-premieres-fois-que-se-passe-t-il-au-sein-du-studio-dubisoft-derriere-trackmania.html#utm_medium=distibuted&utm_source=rss&utm_campaign=646826,1599809025,,unread Comment Konbini s’est fait piéger par un « père masculiniste »,https://www.nouvelobs.com/rue89/20200911.OBS33165/comment-konbini-s-est-fait-pieger-par-un-pere-masculiniste.html,1599819251,,archive -Des abeilles pour résoudre les « conflits » entre les humains et les élépha,https://reporterre.net/Des-abeilles-pour-resoudre-les-conflits-entre-les-humains-et-les-elephants,1599890673,,unread +"Des abeilles pour résoudre les « conflits » entre les humains + +et les élépha",https://reporterre.net/Des-abeilles-pour-resoudre-les-conflits-entre-les-humains-et-les-elephants,1599890673,,unread \ No newline at end of file From 35c4feedd8f473fbe49cd87bab5b073d1c718bd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20L=C5=93uillet?= Date: Wed, 4 Jun 2025 13:42:08 +0200 Subject: [PATCH 07/11] Update joker/graby-site-config --- composer.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/composer.lock b/composer.lock index 2d689bf20..514f40e5b 100644 --- a/composer.lock +++ b/composer.lock @@ -4604,16 +4604,16 @@ }, { "name": "j0k3r/graby-site-config", - "version": "1.0.198", + "version": "1.0.200", "source": { "type": "git", "url": "https://github.com/j0k3r/graby-site-config.git", - "reference": "4ff9bf9ed32186a74d27782a196659d2b319b577" + "reference": "e9cbbc776a7efcc9c06c07ff360744f266bbfc09" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/j0k3r/graby-site-config/zipball/4ff9bf9ed32186a74d27782a196659d2b319b577", - "reference": "4ff9bf9ed32186a74d27782a196659d2b319b577", + "url": "https://api.github.com/repos/j0k3r/graby-site-config/zipball/e9cbbc776a7efcc9c06c07ff360744f266bbfc09", + "reference": "e9cbbc776a7efcc9c06c07ff360744f266bbfc09", "shasum": "" }, "require": { @@ -4642,9 +4642,9 @@ "description": "Graby site config files", "support": { "issues": "https://github.com/j0k3r/graby-site-config/issues", - "source": "https://github.com/j0k3r/graby-site-config/tree/1.0.198" + "source": "https://github.com/j0k3r/graby-site-config/tree/1.0.200" }, - "time": "2025-04-01T02:31:18+00:00" + "time": "2025-06-01T02:42:59+00:00" }, { "name": "j0k3r/httplug-ssrf-plugin", From 4c231963044667a7289de12ee0270d6cb99c60e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20L=C5=93uillet?= Date: Sun, 1 Jun 2025 13:14:07 +0200 Subject: [PATCH 08/11] Prepare 2.6.13 release --- CHANGELOG.md | 15 +++++++++++++++ app/config/wallabag.yml | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d176879ef..eee775fa1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # Changelog +## [2.6.13](https://github.com/wallabag/wallabag/tree/2.6.13) +[Full Changelog](https://github.com/wallabag/wallabag/compare/2.6.12...2.6.13) + +### Improvement + +* Add Pocket and Shaarli import by @nicosomb in [https://github.com/wallabag/wallabag/pull/8193](https://github.com/wallabag/wallabag/pull/8193) + +### Fixes + +* Avoid non-validated OTP to be enabled #8139 by @j0k3r in [https://github.com/wallabag/wallabag/pull/8139](https://github.com/wallabag/wallabag/pull/8139) + +### Technical stuff + +* Add j0k3r/php-readability:1.2.10 as a dependency to fix regression (about latin1 instead of UTF-8 used for entries) by @nicosomb [https://github.com/wallabag/wallabag/pull/8194](https://github.com/wallabag/wallabag/pull/8194) + ## [2.6.12](https://github.com/wallabag/wallabag/tree/2.6.12) [Full Changelog](https://github.com/wallabag/wallabag/compare/2.6.11...2.6.12) diff --git a/app/config/wallabag.yml b/app/config/wallabag.yml index 44cadc57d..34bcd0815 100644 --- a/app/config/wallabag.yml +++ b/app/config/wallabag.yml @@ -1,5 +1,5 @@ wallabag_core: - version: 2.6.12 + version: 2.6.13 paypal_url: "https://liberapay.com/wallabag/donate" languages: en: 'English' From 09af127446c2b569f7d18c011ce590c2336ae3e8 Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Tue, 3 Jun 2025 10:33:41 +0200 Subject: [PATCH 09/11] Unlock php-readability --- composer.json | 5 +---- composer.lock | 16 ++++++++-------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/composer.json b/composer.json index 6f2036df8..65d8e2c17 100644 --- a/composer.json +++ b/composer.json @@ -37,9 +37,6 @@ "email": "hello@wallabag.org", "issues": "https://github.com/wallabag/wallabag/issues" }, - "_comment": [ - "j0k3r/php-readability is pinned to 1.2.10 due to a regression in 1.2.12" - ], "require": { "php": ">=7.4", "ext-ctype": "*", @@ -83,7 +80,7 @@ "html2text/html2text": "^4.3.1", "incenteev/composer-parameter-handler": "^2.1.5", "j0k3r/graby": "^2.4.5", - "j0k3r/php-readability": "1.2.10", + "j0k3r/php-readability": "^1.2.13", "javibravo/simpleue": "^2.1", "jms/serializer": "^3.29.1", "jms/serializer-bundle": "^5.3.1", diff --git a/composer.lock b/composer.lock index 514f40e5b..7e8bd499d 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "931e1540547040b66585959dcbbd4e30", + "content-hash": "f55cde30f67f014d2bf0b589a2fbe5c4", "packages": [ { "name": "babdev/pagerfanta-bundle", @@ -4719,16 +4719,16 @@ }, { "name": "j0k3r/php-readability", - "version": "1.2.10", + "version": "1.2.13", "source": { "type": "git", "url": "https://github.com/j0k3r/php-readability.git", - "reference": "563835730692eb369a73fe4bb9a1b44a603c4ce7" + "reference": "b9dde0f4cd46e9fc082bb37f75dc94ecd2f8faad" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/j0k3r/php-readability/zipball/563835730692eb369a73fe4bb9a1b44a603c4ce7", - "reference": "563835730692eb369a73fe4bb9a1b44a603c4ce7", + "url": "https://api.github.com/repos/j0k3r/php-readability/zipball/b9dde0f4cd46e9fc082bb37f75dc94ecd2f8faad", + "reference": "b9dde0f4cd46e9fc082bb37f75dc94ecd2f8faad", "shasum": "" }, "require": { @@ -4740,7 +4740,7 @@ "require-dev": { "friendsofphp/php-cs-fixer": "^2.14", "monolog/monolog": "^1.24|^2.1", - "symfony/phpunit-bridge": "^4.4|^5.3" + "symfony/phpunit-bridge": "^4.4|^5.3|^6.0|^7.0" }, "suggest": { "ext-tidy": "Used to clean up given HTML and to avoid problems with bad HTML structure." @@ -4790,7 +4790,7 @@ ], "support": { "issues": "https://github.com/j0k3r/php-readability/issues", - "source": "https://github.com/j0k3r/php-readability/tree/1.2.10" + "source": "https://github.com/j0k3r/php-readability/tree/1.2.13" }, "funding": [ { @@ -4798,7 +4798,7 @@ "type": "github" } ], - "time": "2022-06-13T04:15:24+00:00" + "time": "2025-06-03T08:02:58+00:00" }, { "name": "javibravo/simpleue", From 46d6e4d92376da1db2a5f9cf4f2941a15d9a895e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20L=C5=93uillet?= Date: Wed, 4 Jun 2025 11:13:47 +0200 Subject: [PATCH 10/11] Update CHANGELOG --- CHANGELOG.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eee775fa1..f564ee604 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,9 +3,10 @@ ## [2.6.13](https://github.com/wallabag/wallabag/tree/2.6.13) [Full Changelog](https://github.com/wallabag/wallabag/compare/2.6.12...2.6.13) -### Improvement +### Improvements -* Add Pocket and Shaarli import by @nicosomb in [https://github.com/wallabag/wallabag/pull/8193](https://github.com/wallabag/wallabag/pull/8193) +* Add support of Pocket CSV import by @kdecherf and @nicosomb in [https://github.com/wallabag/wallabag/pull/8240](https://github.com/wallabag/wallabag/pull/8240) +* Backport Pocket and Shaarli HTML imports from master by @nicosomb in [https://github.com/wallabag/wallabag/pull/8193](https://github.com/wallabag/wallabag/pull/8193) ### Fixes @@ -13,7 +14,7 @@ ### Technical stuff -* Add j0k3r/php-readability:1.2.10 as a dependency to fix regression (about latin1 instead of UTF-8 used for entries) by @nicosomb [https://github.com/wallabag/wallabag/pull/8194](https://github.com/wallabag/wallabag/pull/8194) +* Update j0k3r/php-readability:1.2.13 to fix regression (about latin1 instead of UTF-8 used for entries) by @nicosomb [https://github.com/wallabag/wallabag/pull/8194](https://github.com/wallabag/wallabag/pull/8194) ## [2.6.12](https://github.com/wallabag/wallabag/tree/2.6.12) [Full Changelog](https://github.com/wallabag/wallabag/compare/2.6.11...2.6.12) From 8e90c0f320404d63134af944c0705af5b10447b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20L=C5=93uillet?= Date: Wed, 4 Jun 2025 17:41:09 +0200 Subject: [PATCH 11/11] Change version in wallabag.yml --- app/config/wallabag.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/config/wallabag.yml b/app/config/wallabag.yml index 34bcd0815..8c1da6b32 100644 --- a/app/config/wallabag.yml +++ b/app/config/wallabag.yml @@ -1,5 +1,5 @@ wallabag_core: - version: 2.6.13 + version: 2.6.14-dev paypal_url: "https://liberapay.com/wallabag/donate" languages: en: 'English'