Mirrors/Frost-for-Facebook
1
0
Fork 0
mirror of https://github.com/AllanWang/Frost-for-Facebook.git synced 2024-11-10 13:02:35 +01:00
Code Issues Releases Activity

Search Parsing (#379)

Browse Source 
* Update parser interface and add search parsing

* Add custom jsoup method and search parse method

* Bind new searchview

* Add search view cache
This commit is contained in:
Allan Wang 2017-10-11 01:51:21 -04:00 committed by GitHub
parent d12e0697ad
commit fe1df730a1
14 changed files with 199 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
app/src/main/kotlin/com/pitchedapps/frost/activities/MainActivity.kt
View File

@ -27,7 +27,6 @@ import ca.allanwang.kau.searchview.SearchItem
import ca.allanwang.kau.searchview.SearchView
import ca.allanwang.kau.searchview.bindSearchView
import ca.allanwang.kau.utils.*
import ca.allanwang.kau.xml.showChangelog
import co.zsmb.materialdrawerkt.builders.Builder
import co.zsmb.materialdrawerkt.builders.accountHeader
import co.zsmb.materialdrawerkt.builders.drawer
@ -54,21 +53,23 @@ import com.pitchedapps.frost.facebook.FbCookie.switchUser
import com.pitchedapps.frost.facebook.FbItem
import com.pitchedapps.frost.facebook.PROFILE_PICTURE_URL
import com.pitchedapps.frost.fragments.WebFragment
import com.pitchedapps.frost.parsers.SearchParser
import com.pitchedapps.frost.utils.*
import com.pitchedapps.frost.utils.iab.FrostBilling
import com.pitchedapps.frost.utils.iab.IABMain
import com.pitchedapps.frost.utils.iab.IS_FROST_PRO
import com.pitchedapps.frost.views.BadgedIcon
import com.pitchedapps.frost.views.FrostViewPager
import com.pitchedapps.frost.web.SearchWebView
import io.reactivex.android.schedulers.AndroidSchedulers
import io.reactivex.disposables.Disposable
import io.reactivex.schedulers.Schedulers
import io.reactivex.subjects.PublishSubject
import org.jetbrains.anko.doAsync
import org.jetbrains.anko.uiThread
import org.jsoup.Jsoup
import java.util.concurrent.TimeUnit
class MainActivity : BaseActivity(), SearchWebView.SearchContract,
class MainActivity : BaseActivity(),
ActivityWebContract, FileChooserContract by FileChooserDelegate(),
FrostBilling by IABMain() {
@ -84,19 +85,14 @@ class MainActivity : BaseActivity(), SearchWebView.SearchContract,
var webFragmentObservable = PublishSubject.create<Int>()!!
var lastPosition = -1
val headerBadgeObservable = PublishSubject.create<String>()
var hiddenSearchView: SearchWebView? = null
var firstLoadFinished = false
set(value) {
if (field && value) return //both vals are already true
L.i("First fragment load has finished")
field = value
if (value && hiddenSearchView == null) {
hiddenSearchView = SearchWebView(this, this)
}
}
var searchView: SearchView? = null
override val isSearchOpened: Boolean
get() = searchView?.isOpen ?: false
val searchViewCache = mutableMapOf<String, List<SearchItem>>()
companion object {
const val ACTIVITY_SETTINGS = 97
@ -329,20 +325,6 @@ class MainActivity : BaseActivity(), SearchWebView.SearchContract,
onClick { _ -> onClick(); false }
}
/**
* Something happened where the normal search function won't work
* Fallback to overlay style
*/
override fun disposeHeadlessSearch() {
hiddenSearchView = null
searchView?.config { textCallback = { _, _ -> } }
}
override fun emitSearchResponse(items: List<SearchItem>) {
searchView?.results = items
}
fun refreshAll() {
webFragmentObservable.onNext(WebFragment.REQUEST_REFRESH)
}
@ -353,20 +335,25 @@ class MainActivity : BaseActivity(), SearchWebView.SearchContract,
setMenuIcons(menu, Prefs.iconColor,
R.id.action_settings to GoogleMaterial.Icon.gmd_settings,
R.id.action_search to GoogleMaterial.Icon.gmd_search)
if (Prefs.searchBar) {
if (firstLoadFinished && hiddenSearchView == null) hiddenSearchView = SearchWebView(this, this)
if (searchView == null) searchView = bindSearchView(menu, R.id.action_search, Prefs.iconColor) {
textCallback = { query, _ -> runOnUiThread { hiddenSearchView?.query(query) } }
searchCallback = { query, _ -> launchWebOverlay("${FbItem.SEARCH.url}/?q=$query"); true }
foregroundColor = Prefs.textColor
backgroundColor = Prefs.bgColor.withMinAlpha(200)
openListener = { hiddenSearchView?.pauseLoad = false }
closeListener = { hiddenSearchView?.pauseLoad = true }
onItemClick = { _, key, _, _ -> launchWebOverlay(key) }
if (searchView == null) searchView = bindSearchView(menu, R.id.action_search, Prefs.iconColor) {
textCallback = { query, _ ->
val results = searchViewCache[query]
if (results != null)
runOnUiThread { searchView?.results = results }
else
doAsync {
val data = SearchParser.query(query) ?: return@doAsync
val items = data.map { SearchItem(it.href, it.title, it.description) }
searchViewCache.put(query, items)
uiThread { searchView?.results = items }
}
}
} else {
if (searchView != null) disposeHeadlessSearch()
else menu.findItem(R.id.action_search).setOnMenuItemClickListener { _ -> launchWebOverlay(FbItem.SEARCH.url); true }
textDebounceInterval = 300
searchCallback = { query, _ -> launchWebOverlay("${FbItem.SEARCH.url}/?q=$query"); true }
closeListener = { _ -> searchViewCache.clear() }
foregroundColor = Prefs.textColor
backgroundColor = Prefs.bgColor.withMinAlpha(200)
onItemClick = { _, key, _, _ -> launchWebOverlay(key) }
}
return true
}
@ -438,7 +425,7 @@ class MainActivity : BaseActivity(), SearchWebView.SearchContract,
}
override fun onBackPressed() {
if (searchView?.onBackPressed() ?: false) return
if (searchView?.onBackPressed() == true) return
if (currentFragment.onBackPressed()) return
super.onBackPressed()
}

5
app/src/main/kotlin/com/pitchedapps/frost/dbflow/CookiesDb.kt
View File

@ -6,6 +6,7 @@ import com.github.pwittchen.reactivenetwork.library.rx2.ReactiveNetwork
import com.pitchedapps.frost.facebook.FACEBOOK_COM
import com.pitchedapps.frost.facebook.FbItem
import com.pitchedapps.frost.utils.L
import com.pitchedapps.frost.utils.frostJsoup
import com.pitchedapps.frost.utils.logFrostAnswers
import com.raizlabs.android.dbflow.annotation.ConflictAction
import com.raizlabs.android.dbflow.annotation.Database
@ -71,9 +72,7 @@ fun CookieModel.fetchUsername(callback: (String) -> Unit) {
if (!yes) return@subscribe callback("")
var result = ""
try {
result = Jsoup.connect(FbItem.PROFILE.url)
.cookie(FACEBOOK_COM, cookie)
.get().title()
result = frostJsoup(cookie, FbItem.PROFILE.url).title()
L.d("Fetch username found", result)
} catch (e: Exception) {
if (e !is UnknownHostException)

57
app/src/main/kotlin/com/pitchedapps/frost/parsers/FrostParser.kt
View File

@ -1,33 +1,78 @@
package com.pitchedapps.frost.parsers
import org.jsoup.nodes.Document
/**
* Created by Allan Wang on 2017-10-06.
*
* Interface for a given parser
* Use cases should be attached as delegates to objects that implement this interface
*
* In all cases, parsing will be done from a JSoup document
* Variants accepting strings are also permitted, and they will be converted to documents accordingly
*/
interface FrostParser<T> {
/**
* Extracts data from the JSoup document
* In some cases, the document can be created directly from a connection
* In other times, it needs to be created from scripts, which otherwise
* won't be parsed
*/
fun parse(doc: Document): T?
/**
* Parse a String input
*/
fun parse(text: String?): T?
/**
* Take in doc and emit debug output
*/
fun debug(doc: Document): String
/**
* Attempts to parse input and emit a debugger
*/
fun debug(text: String?): String
}
internal abstract class FrostParserBase<T> : FrostParser<T> {
override final fun parse(text: String?): T?
= if (text == null) null else parseImpl(text)
override final fun parse(text: String?): T? {
text ?: return null
val doc = textToDoc(text) ?: return null
return parse(doc)
}
protected abstract fun parseImpl(text: String): T?
protected abstract fun textToDoc(text: String): Document?
override final fun debug(text: String?): String {
override fun debug(text: String?): String {
val result = mutableListOf<String>()
result.add("Testing parser for ${this::class.java.simpleName}")
if (text == null) {
result.add("Input is null")
result.add("Null text input")
return result.joinToString("\n")
}
val output = parseImpl(text)
val doc = textToDoc(text)
if (doc == null) {
result.add("Null document from text")
return result.joinToString("\n")
}
return debug(doc, result)
}
override final fun debug(doc: Document): String {
val result = mutableListOf<String>()
result.add("Testing parser for ${this::class.java.simpleName}")
return debug(doc, result)
}
private fun debug(doc: Document, result: MutableList<String>): String {
val output = parse(doc)
if (output == null) {
result.add("Output is null")
return result.joinToString("\n")
} else {
result.add("Output is not null")
}
debugImpl(output, result)
return result.joinToString("\n")

16
app/src/main/kotlin/com/pitchedapps/frost/parsers/MessageParser.kt
View File

@ -5,6 +5,7 @@ import com.pitchedapps.frost.facebook.formattedFbUrlCss
import com.pitchedapps.frost.utils.L
import org.apache.commons.text.StringEscapeUtils
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
/**
@ -22,7 +23,7 @@ data class FrostLink(val text: String, val href: String)
private class MessageParserImpl : FrostParserBase<Triple<List<FrostThread>, FrostLink?, List<FrostLink>>>() {
override fun parseImpl(text: String): Triple<List<FrostThread>, FrostLink?, List<FrostLink>>? {
override fun textToDoc(text: String): Document? {
var content = StringEscapeUtils.unescapeEcmaScript(text)
val begin = content.indexOf("id=\"threadlist_rows\"")
if (begin <= 0) {
@ -36,11 +37,14 @@ private class MessageParserImpl : FrostParserBase<Triple<List<FrostThread>, Fros
return null
}
content = content.substring(0, end).substringBeforeLast("</div>")
val body = Jsoup.parseBodyFragment("<div $content")
val threadList = body.getElementById("threadlist_rows")
return Jsoup.parseBodyFragment("<div $content")
}
override fun parse(doc: Document): Triple<List<FrostThread>, FrostLink?, List<FrostLink>>? {
val threadList = doc.getElementById("threadlist_rows")
val threads: List<FrostThread> = threadList.getElementsByAttributeValueContaining("id", "thread_fbid_")
.mapNotNull { parseMessage(it) }
val seeMore = parseLink(body.getElementById("see_older_threads"))
val seeMore = parseLink(doc.getElementById("see_older_threads"))
val extraLinks = threadList.nextElementSibling().select("a")
.mapNotNull { parseLink(it) }
return Triple(threads, seeMore, extraLinks)
@ -76,9 +80,9 @@ private class MessageParserImpl : FrostParserBase<Triple<List<FrostThread>, Fros
}
override fun debugImpl(data: Triple<List<FrostThread>, FrostLink?, List<FrostLink>>, result: MutableList<String>) {
result.addAll(data.first.map { it.toString() })
result.addAll(data.first.map(FrostThread::toString))
result.add("See more link:")
result.add("\t${data.second}")
result.addAll(data.third.map { it.toString() })
result.addAll(data.third.map(FrostLink::toString))
}
}

73
app/src/main/kotlin/com/pitchedapps/frost/parsers/SearchParser.kt Normal file
View File

@ -0,0 +1,73 @@
package com.pitchedapps.frost.parsers
import ca.allanwang.kau.utils.withMaxLength
import com.pitchedapps.frost.facebook.FbItem
import com.pitchedapps.frost.facebook.formattedFbUrlCss
import com.pitchedapps.frost.utils.L
import com.pitchedapps.frost.utils.frostJsoup
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
/**
* Created by Allan Wang on 2017-10-09.
*/
object SearchParser : FrostParser<List<FrostSearch>> by SearchParserImpl() {
fun query(input: String): List<FrostSearch>? {
val url = "${FbItem.SEARCH.url}?q=$input"
L.i(null, "Search Query $url")
return parse(frostJsoup(url))
}
}
enum class SearchKeys(val key: String) {
USERS("keywords_users"),
EVENTS("keywords_events")
}
/**
* As far as I'm aware, all links are independent, so the queries don't matter
* A lot of it is tracking information, which I'll strip away
* Other text items are formatted for safety
*/
class FrostSearch(href: String, title: String, description: String?) {
val href = with(href.indexOf("?")) { if (this == -1) href else href.substring(0, this) }
val title = title.format()
val description = description?.format()
private fun String.format() = replace("\n", " ").withMaxLength(50)
override fun toString(): String
= "FrostSearch(href=$href, title=$title, description=$description)"
}
private class SearchParserImpl : FrostParserBase<List<FrostSearch>>() {
override fun parse(doc: Document): List<FrostSearch>? {
val container: Element = doc.getElementById("BrowseResultsContainer")
?: doc.getElementById("root")
?: return null
val hrefSet = mutableSetOf<String>()
/**
* When mapping items, some links are duplicated because they are nested below a main one
* We will filter out search items whose links are already in the list
*
* Removed [data-store*=result_id]
*/
return container.select("a.touchable.primary[href]").filter(Element::hasText).mapNotNull {
val item = FrostSearch(it.attr("href").formattedFbUrlCss,
it.select("._uok").first()?.text() ?: it.text(),
it.select("._1tcc").first()?.text())
if (hrefSet.contains(item.href)) return@mapNotNull null
hrefSet.add(item.href)
item
}
}
override fun textToDoc(text: String): Document? = Jsoup.parse(text)
override fun debugImpl(data: List<FrostSearch>, result: MutableList<String>) {
result.addAll(data.map(FrostSearch::toString))
}
}

8
app/src/main/kotlin/com/pitchedapps/frost/services/NotificationService.kt
View File

@ -10,16 +10,14 @@ import com.pitchedapps.frost.R
import com.pitchedapps.frost.dbflow.CookieModel
import com.pitchedapps.frost.dbflow.lastNotificationTime
import com.pitchedapps.frost.dbflow.loadFbCookiesSync
import com.pitchedapps.frost.facebook.FACEBOOK_COM
import com.pitchedapps.frost.facebook.FbItem
import com.pitchedapps.frost.facebook.USER_AGENT_BASIC
import com.pitchedapps.frost.facebook.formattedFbUrl
import com.pitchedapps.frost.parsers.MessageParser
import com.pitchedapps.frost.utils.L
import com.pitchedapps.frost.utils.Prefs
import com.pitchedapps.frost.utils.frostAnswersCustom
import com.pitchedapps.frost.utils.frostJsoup
import org.jetbrains.anko.doAsync
import org.jsoup.Jsoup
import org.jsoup.nodes.Element
import java.util.concurrent.Future
@ -101,7 +99,7 @@ class NotificationService : JobService() {
fun fetchGeneralNotifications(data: CookieModel) {
L.d("Notif fetch", data.toString())
val doc = Jsoup.connect(FbItem.NOTIFICATIONS.url).cookie(FACEBOOK_COM, data.cookie).userAgent(USER_AGENT_BASIC).get()
val doc = frostJsoup(data.cookie, FbItem.NOTIFICATIONS.url)
//aclb for unread, acw for read
val unreadNotifications = (doc.getElementById("notifications_list") ?: return L.eThrow("Notification list not found")).getElementsByClass("aclb")
var notifCount = 0
@ -149,7 +147,7 @@ class NotificationService : JobService() {
fun fetchMessageNotifications(data: CookieModel) {
L.d("Notif IM fetch", data.toString())
val doc = Jsoup.connect(FbItem.MESSAGES.url).cookie(FACEBOOK_COM, data.cookie).userAgent(USER_AGENT_BASIC).get()
val doc = frostJsoup(data.cookie, FbItem.MESSAGES.url)
val (threads, _, _) = MessageParser.parse(doc.toString()) ?: return L.e("Could not parse IM")
var notifCount = 0

4
app/src/main/kotlin/com/pitchedapps/frost/settings/Behaviour.kt
View File

@ -31,10 +31,6 @@ fun SettingsActivity.getBehaviourPrefs(): KPrefAdapterBuilder.() -> Unit = {
descRes = R.string.viewpager_swipe_desc
}
checkbox(R.string.search_bar, { Prefs.searchBar }, { Prefs.searchBar = it; setFrostResult(MainActivity.REQUEST_SEARCH) }) {
descRes = R.string.search_bar_desc
}
checkbox(R.string.force_message_bottom, { Prefs.messageScrollToBottom }, { Prefs.messageScrollToBottom = it }) {
descRes = R.string.force_message_bottom_desc
}

11
app/src/main/kotlin/com/pitchedapps/frost/settings/Debug.kt
View File

@ -14,10 +14,7 @@ import com.pitchedapps.frost.facebook.USER_AGENT_BASIC
import com.pitchedapps.frost.injectors.InjectorContract
import com.pitchedapps.frost.injectors.JsActions
import com.pitchedapps.frost.injectors.JsAssets
import com.pitchedapps.frost.utils.L
import com.pitchedapps.frost.utils.cleanHtml
import com.pitchedapps.frost.utils.materialDialogThemed
import com.pitchedapps.frost.utils.sendFrostEmail
import com.pitchedapps.frost.utils.*
import com.pitchedapps.frost.web.launchHeadlessHtmlExtractor
import com.pitchedapps.frost.web.query
import io.reactivex.disposables.Disposable
@ -119,11 +116,7 @@ private enum class Debugger(val data: FbItem, val injector: InjectorContract?, v
uiThread {
it.setContent("Load Jsoup")
it.setOnCancelListener(null)
it.debugAsync {
val connection = Jsoup.connect(data.url).cookie(FACEBOOK_COM, FbCookie.webCookie).userAgent(USER_AGENT_BASIC)
val doc = connection.get()
simplifyJsoup(doc)
}
it.debugAsync { simplifyJsoup(frostJsoup(data.url)) }
}
}

4
app/src/main/kotlin/com/pitchedapps/frost/utils/Prefs.kt
View File

@ -134,8 +134,6 @@ object Prefs : KPref() {
var analytics: Boolean by kpref("analytics", true)
var searchBar: Boolean by kpref("search_bar", true)
var overlayEnabled: Boolean by kpref("overlay_enabled", true)
var overlayFullScreenSwipe: Boolean by kpref("overlay_full_screen_swipe", true)
@ -152,4 +150,6 @@ object Prefs : KPref() {
val mainActivityLayout: MainActivityLayout
get() = MainActivityLayout(mainActivityLayoutType)
override fun deleteKeys() = arrayOf("search_bar")
}

10
app/src/main/kotlin/com/pitchedapps/frost/utils/Utils.kt
View File

@ -29,11 +29,9 @@ import com.pitchedapps.frost.BuildConfig
import com.pitchedapps.frost.R
import com.pitchedapps.frost.activities.*
import com.pitchedapps.frost.dbflow.CookieModel
import com.pitchedapps.frost.facebook.FACEBOOK_COM
import com.pitchedapps.frost.facebook.FbCookie
import com.pitchedapps.frost.facebook.FbItem
import com.pitchedapps.frost.facebook.formattedFbUrl
import com.pitchedapps.frost.facebook.*
import com.pitchedapps.frost.utils.iab.IS_FROST_PRO
import org.jsoup.Jsoup
import java.io.IOException
import java.util.*
@ -219,5 +217,9 @@ inline fun Context.sendFrostEmail(subjectId: String, crossinline builder: EmailB
addItem("Random Frost ID", "${Prefs.frostId}-$proTag")
}
fun frostJsoup(url: String)
= frostJsoup(FbCookie.webCookie, url)
fun frostJsoup(cookie: String?, url: String)
= Jsoup.connect(url).cookie(FACEBOOK_COM, cookie).userAgent(USER_AGENT_BASIC).get()!!

10
app/src/test/kotlin/com/pitchedapps/frost/Base.java
View File

@ -1,10 +0,0 @@
package com.pitchedapps.frost;
/**
* Created by Allan Wang on 2017-10-06.
*
* Empty class to hold a reference to the target output
*/
public class Base {
}

5
app/src/test/kotlin/com/pitchedapps/frost/parsers/MessageParserTest.kt
View File

@ -10,10 +10,7 @@ import kotlin.test.assertEquals
class MessageParserTest {
@Test
fun basic() {
val content = getResource("priv/messages.html") ?: return
println(MessageParser.debug(content))
}
fun basic() = debug("messages", MessageParser)
@Test
fun parseEpoch() {

10
app/src/test/kotlin/com/pitchedapps/frost/parsers/ParserTestHelper.kt
View File

@ -1,18 +1,22 @@
package com.pitchedapps.frost.parsers
import com.pitchedapps.frost.Base
import java.net.URL
import java.nio.file.Paths
/**
* Created by Allan Wang on 2017-10-06.
*/
fun <T : Any> T.getResource(path: String): String? {
fun <T : Any> T.getResource(path: String): String? {
Paths.get("src/test/resources/${path.trimStart('/')}")
val resource: URL? = Base::class.java.classLoader.getResource(path)
val resource: URL? = this::class.java.classLoader.getResource(path)
if (resource == null) {
println("Resource at $path could not be found")
return null
}
return resource.readText()
}
fun <T : Any, P : Any> T.debug(path: String, parser: FrostParser<P>) {
val content = getResource("priv/$path.html") ?: return
println(parser.debug(content))
}

18
app/src/test/kotlin/com/pitchedapps/frost/parsers/SearchParserTest.kt Normal file
View File

@ -0,0 +1,18 @@
package com.pitchedapps.frost.parsers
import org.junit.Test
/**
* Created by Allan Wang on 2017-10-06.
*/
class SearchParserTest {
@Test
fun debug() = debug("search", SearchParser)
@Test
fun debug2() = debug("search2", SearchParser)
@Test
fun debug3() = debug("search3", SearchParser)
}