stanikol / olx Goto Github PK

        siteid = Try("""(?:ID ogłoszenia\:\s*)(\d+)""".r.findFirstMatchIn(soup.select("span:matches((Dodane|Dodane z telefonu))").head.text()).get.group(1)).toOption.getOrElse("")
        //
        phoneToken = getPhoneToken(responseBody)
        cookies = getCookies(response).map {
          case HttpCookie(name, value, _, _, _, _, _, _, _) if name.equals("pt") =>
            HttpCookiePair(name, phoneToken)
          case x@HttpCookie(name, value, _, _, _, _, _, _, _) =>
            HttpCookiePair(name, value)
        }
        //
        data = Map(
          "seq" -> counter.get().toString,
          "siteid" -> siteid,
          "brief" -> Try(soup.select("table:contains(Oferta od) > tbody > tr > td").map(_.text).mkString("\n")).toOption.getOrElse(""),
          "head" -> soup.select(".offer-titlebox h1").headOption.map(_.text().trim).getOrElse(""),
          "text" -> soup.select("#textContent").headOption.map(_.text()).getOrElse(""),
          "pubdate" -> soup.select("span:matches((Dodane|Dodane z telefonu))").headOption.map(_.text()).getOrElse(""),
          "url" -> uri,
          "usrid" -> Try(usrid).toOption.getOrElse(""),
          "user" -> Try(soup.select("a:matches((?i)(Ogłoszenia użytkownika|Ogłoszenia użytkownika))").head.attr("href")).getOrElse(""),
          "price" -> soup.select("div.pricelabel strong").headOption.map(_.text()).getOrElse(""),
          "location" -> soup.select(".show-map-link").headOption.map(_.text()).getOrElse(""),
          "username" -> soup.select(".userdetails span").headOption.map(_.text()).getOrElse(""),
          "section" -> Try(soup.select("#breadcrumbTop ul span").map(_.text()).filter(_.nonEmpty).mkString("/")).toOption.getOrElse(""),
          "viewed" -> Try(soup.select("div.pdingtop10:contains(Wyświetleń:) > strong").head.text()).toOption.getOrElse(""),
          "downdate" -> DateTime.now().toString("yyyy-MM-dd HHmmss")
        )
      } yield (data, Cookie(cookies), phoneToken, usrid)
    }
  }

  def downloadPhones(implicit as: ActorSystem, mat: ActorMaterializer, ec: ExecutionContext) =
    Flow[(Map[String, String], Cookie, String, String)].mapAsync(10){ case (data, cookie, phoneToken, usrid) =>
      val phonesUri = s"https://www.olx.pl/ajax/misc/contact/phone/$usrid/?pt=${phoneToken}"