vendor/nen/kennisbank-platform/src/Nen/Publication/Indexer.php line 103

Open in your IDE?
  1. <?php namespace Nen\Bundle\KennisbankPlatformBundle\Publication;
  3. use Doctrine\ORM\EntityManagerInterface;
  4. use DOMDocument;
  5. use DOMElement;
  6. use DOMNode;
  7. use DOMXPath;
  8. use Exception;
  9. use Nen\Bundle\KennisbankPlatformBundle\Bolt\BoltConfiguration;
  10. use Nen\Bundle\KennisbankPlatformBundle\Entity\BoltPublication;
  11. use Nen\Bundle\KennisbankPlatformBundle\Entity\EsDocument;
  12. use Nen\Bundle\KennisbankPlatformBundle\Entity\PublisherEntry;
  13. use Nen\Bundle\KennisbankPlatformBundle\Exceptions\SectionNotFoundException;
  14. use Nen\Bundle\KennisbankPlatformBundle\Publication\Event\PublisherFinishEvent;
  15. use Nen\Bundle\KennisbankPlatformBundle\Repository\EsDocumentRepository;
  16. use Psr\Log\LoggerInterface;
  17. use Smalot\PdfParser\Parser;
  18. use Symfony\Component\EventDispatcher\EventSubscriberInterface;
  19. use Symfony\Component\Filesystem\Filesystem;
  21. class Indexer implements EventSubscriberInterface
  22. {
  25.     /**
  26.      * @var string
  27.      */
  28.     protected $path;
  31.     /**
  32.      * @var EsDocumentRepository
  33.      */
  34.     protected $repository;
  37.     /**
  38.      * @var Publisher
  39.      */
  40.     protected $publisher;
  43.     /**
  44.      * @var EntityManagerInterface
  45.      */
  46.     protected $em;
  49.     /**
  50.      * @var Filesystem
  51.      */
  52.     protected $fs;
  55.     /**
  56.      * @var LoggerInterface
  57.      */
  58.     private $logger;
  61.     /**
  62.      * @var BoltConfiguration
  63.      */
  64.     private $configuration;
  67.     /**
  68.      * @param EntityManagerInterface $manager
  69.      * @param Publisher              $publisher
  70.      * @param EsDocumentRepository   $repository
  71.      * @param BoltConfiguration      $configuration
  72.      */
  73.     public function __construct(
  74.         EntityManagerInterface $manager,
  75.         Publisher $publisher,
  76.         EsDocumentRepository $repository,
  77.         BoltConfiguration $configuration
  78.     ) {
  79.         $this->em            = $manager;
  80.         $this->publisher     = $publisher;
  81.         $this->repository    = $repository;
  82.         $this->logger        = $publisher->getLogger();
  83.         $this->fs            = $publisher->getFileSystem();
  84.         $this->configuration = $configuration;
  85.     }
  88.     /**
  89.      * @return array
  90.      */
  91.     public static function getSubscribedEvents()
  92.     {
  93.         return [
  94.             PublisherFinishEvent::class => 'onPublicationFinish',
  95.         ];
  96.     }
  99.     /**
  100.      * @param PublisherFinishEvent $event
  101.      * @throws Exception
  102.      */
  103.     public function onPublicationFinish(PublisherFinishEvent $event)
  104.     {
  105.         $this->index($event->getPublication(), $event->getEntry());
  106.     }
  109.     /**
  110.      * @param BoltPublication     $publication
  111.      * @param PublisherEntry|null $entry
  112.      * @throws Exception
  113.      */
  114.     public function index(BoltPublication $publication, PublisherEntry $entry = null)
  115.     {
  116.         if ($publication->getZip() || $publication->hasPdf()) {
  117.             $this->logger->info('Creating index for '.$publication->getTitle());
  118.         } else {
  119.             $this->logger->info('Skipping publication '.$publication->getTitle().' it is not indexable.');
  121.             $this->repository->removeForPublication($publication);
  123.             return;
  124.         }
  126.         $this->repository->removeForPublication($publication);
  128.         if ($publication->getStatus() === 'published') {
  129.             if ($publication->getZip()) {
  130.                 $this->path = $this->createAndReturnIndexPath($publication);
  132.                 $sections = $publication->getSections();
  134.                 foreach ($sections as $section) {
  135.                     if ($section === 'TOC') {
  136.                         continue;
  137.                     }
  139.                     $this->indexSection(trim($section), $publication);
  140.                 }
  141.             } else {
  142.                 $this->indexPdf($publication);
  143.             }
  144.         }
  146.         $this->logger->info('Finished creating index');
  147.     }
  150.     /**
  151.      * @param BoltPublication     $publication
  152.      * @param PublisherEntry|null $entry
  153.      * @throws Exception
  154.      */
  155.     public function remove(BoltPublication $publication, PublisherEntry $entry = null)
  156.     {
  158.         $this->repository->removeForPublication($publication);
  159.     }
  162.     /**
  163.      * @param BoltPublication $publication
  164.      * @return string
  165.      */
  166.     private function createAndReturnIndexPath(BoltPublication $publication): string
  167.     {
  168.         $path = $this->publisher->getPublicationFolder($publication).'/index';
  170.         if ($this->fs->exists($path)) {
  171.             $this->fs->remove($path);
  172.         }
  174.         $this->fs->mkdir($path);
  176.         return $path;
  177.     }
  180.     /**
  181.      * @param string          $section
  182.      * @param BoltPublication $publication
  183.      */
  184.     public function indexSection(string $section, BoltPublication $publication)
  185.     {
  186.         set_time_limit(30);
  188.         try {
  189.             $html = $this->publisher->getPublishedSection($section, $publication);
  190.         } catch (SectionNotFoundException $e) {
  191.             return;
  192.         }
  194.         $document = new DOMDocument('1.0', 'UTF-8');
  196.         libxml_use_internal_errors(true);
  197.         $document->loadHTML('<?xml encoding="UTF-8">'.$html);
  198.         libxml_use_internal_errors(false);
  200.         $xpath = new DOMXPath($document);
  202.         $nodes = $xpath->query('//div[@class="sts-section"]');
  204.         if ($nodes->length === 0) {
  205.             $nodes = $xpath->query('//div[@class="sts-app"]');
  206.         }
  208.         if ($nodes->length === 0) {
  209.             return;
  210.         }
  212.         /** @var DOMElement $section */
  213.         foreach ($nodes as $node) {
  214.             $local  = $node->getAttribute('id');
  215.             $html   = $document->saveHTML($node);
  216.             $id     = $publication->getId().'_'.$section.'_'.$local;
  217.             $parsed = $this->parseSection($local, $node, $xpath);
  219.             $this->fs->dumpFile(
  220.                 $this->path.'/'.$section.'_'.$local.'.html',
  221.                 $html
  222.             );
  224.             if (!$esDocument = $this->repository->findOneBy(['documentId' => $id])) {
  225.                 $esDocument = new EsDocument();
  226.             }
  228.             $meta = [
  229.                 'publication' => [
  230.                     'id'             => $publication->getId(),
  231.                     'slug'           => $publication->getSlug(),
  232.                     'title'          => $publication->getTitle(),
  233.                     'short_title'    => $publication->getShortTitle() ?: $publication->getTitle(),
  234.                     'category'       => $publication->getCategory(),
  235.                     'category_label' => $this->configuration->getContentTypeSelectValues('publications',
  236.                         'category')[$publication->getCategory()],
  237.                     'status'         => $publication->getPublicationStatus(),
  238.                 ],
  239.                 'section'     => $section,
  240.                 'local'       => $local,
  241.                 'number'      => $parsed['number'],
  242.             ];
  244.             $esDocument
  245.                 ->setDocumentId($id)
  246.                 ->setHtml($html)
  247.                 ->setTitle($parsed['title'])
  248.                 ->setText($parsed['text'])
  249.                 ->setPublication($publication)
  250.                 ->setMetadata($meta);
  252.             $this->em->persist($esDocument);
  253.         }
  255.         $this->em->flush();
  256.     }
  259.     /**
  260.      * @param string   $local
  261.      * @param DOMNode  $node
  262.      * @param DOMXPath $xpath
  263.      * @return array
  264.      */
  265.     protected function parseSection(string $local, DOMNode $node, DOMXPath $xpath): array
  266.     {
  267.         $number = str_replace('sec_', '', $local);
  268.         $title  = null;
  269.         $text   = null;
  271.         $nodes = $xpath->query('.//*[@class="sts-sec-title" or @class="sts-section-label"]', $node);
  273.         if ($nodes->length > 0) {
  274.             $title = $nodes->item(0)->nodeValue;
  276.             $nodes->item(0)->parentNode->removeChild($nodes->item(0));
  277.         }
  279.         if (empty($title)) {
  280.             $title = $number;
  281.         }
  283.         $text = preg_replace('/\\[nrt]+/', ' ', strip_tags($node->nodeValue));
  284.         $text = preg_replace('/[ ]{2,}/', ' ', $text);
  286.         $title = preg_replace('/[\x{00A0}]+$/u', '', $title);
  287.         $title = preg_replace('/\\[nrt]+/', ' ', $title);
  288.         $title = preg_replace('/[ ]{2,}/', ' ', $title);
  290.         return [
  291.             'number' => (string)$number,
  292.             'title'  => trim($title),
  293.             'text'   => trim($text),
  294.         ];
  295.     }
  298.     /**
  299.      * @param BoltPublication $publication
  300.      * @throws Exception
  301.      */
  302.     private function indexPdf(BoltPublication $publication)
  303.     {
  304.         $file = $publication->getPdfForIndex();
  306.         $this->logger->info('Processing file ' . $file);
  308.         $parser = new Parser();
  310.         try {
  311.             $pdf = $parser->parseFile($this->configuration->getFilesPath().$publication->getPdf());
  312.         } catch (Exception $e) {
  313.             $this->logger->error($e->getMessage());
  315.             return;
  316.         }
  318.         $pages  = $pdf->getPages();
  319.         $number = 1;
  321.         foreach ($pages as $page) {
  322.             $id = $publication->getId().$number;
  324.             if (!$esDocument = $this->repository->findOneBy(['documentId' => $id])) {
  325.                 $esDocument = new EsDocument();
  326.             }
  328.             $meta = [
  329.                 'publication' => [
  330.                     'id'             => $publication->getId(),
  331.                     'slug'           => $publication->getSlug(),
  332.                     'title'          => $publication->getTitle(),
  333.                     'short_title'    => $publication->getShortTitle() ?: $publication->getTitle(),
  334.                     'category'       => $publication->getCategory(),
  335.                     'category_label' => $this->configuration->getContentTypeSelectValues(
  336.                         'publications', 'category')[$publication->getCategory()],
  337.                     'status'         => $publication->getPublicationStatus(),
  338.                 ],
  339.                 'section'     => $number,
  340.                 'local'       => $number,
  341.                 'number'      => $number,
  342.             ];
  344.             $esDocument
  345.                 ->setDocumentId($id)
  346.                 ->setHtml($page->getText())
  347.                 ->setTitle($publication->getTitle())
  348.                 ->setText($page->getText())
  349.                 ->setPublication($publication)
  350.                 ->setMetadata($meta);
  352.             $this->em->persist($esDocument);
  354.             $this->em->flush();
  356.             $number++;
  357.         }
  358.     }
  359. }