From de4d6037d35ef540ff9307e08a4c04c2c5b8dd81 Mon Sep 17 00:00:00 2001 From: Vincent Nagel Date: Sun, 15 Mar 2020 16:53:29 -0500 Subject: [PATCH] ageRestrictedContent first draft Cookie updated whenever ageRestrictedContent setting is changed or service is changed. Right now there is only a cookie for youtube, but cookies for other services could be added in the future. Problems with this approach: Even when the service is set to youtube, the downloader doesn't only request youtube urls e.g. it also sends reqeusts to i.ytimg.com, suggestqueries.google.com, and yt3.ggpht.com. The ageRestrictedContent cookie is not normally sent when sending requests to these other urls, so doing so might have unknown effects. --- .../java/org/schabi/newpipe/DebugApp.java | 1 + app/src/main/java/org/schabi/newpipe/App.java | 3 +- .../org/schabi/newpipe/DownloaderImpl.java | 52 +++++++++++++++---- .../java/org/schabi/newpipe/MainActivity.java | 1 + .../org/schabi/newpipe/ReCaptchaActivity.java | 3 +- .../settings/ContentSettingsFragment.java | 11 ++++ .../org/schabi/newpipe/util/CookieUtils.java | 32 ++++++++++++ 7 files changed, 92 insertions(+), 11 deletions(-) create mode 100644 app/src/main/java/org/schabi/newpipe/util/CookieUtils.java diff --git a/app/src/debug/java/org/schabi/newpipe/DebugApp.java b/app/src/debug/java/org/schabi/newpipe/DebugApp.java index 6bcf71035..a378911c6 100644 --- a/app/src/debug/java/org/schabi/newpipe/DebugApp.java +++ b/app/src/debug/java/org/schabi/newpipe/DebugApp.java @@ -43,6 +43,7 @@ public class DebugApp extends App { DownloaderImpl downloader = DownloaderImpl.init(new OkHttpClient.Builder() .addNetworkInterceptor(new StethoInterceptor())); setCookiesToDownloader(downloader); + downloader.updateAgeRestrictedContentCookies(getApplicationContext()); return downloader; } diff --git a/app/src/main/java/org/schabi/newpipe/App.java b/app/src/main/java/org/schabi/newpipe/App.java index 4d05c69cc..167f459f0 100644 --- a/app/src/main/java/org/schabi/newpipe/App.java +++ b/app/src/main/java/org/schabi/newpipe/App.java @@ -136,7 +136,8 @@ public class App extends Application { final SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences( getApplicationContext()); final String key = getApplicationContext().getString(R.string.recaptcha_cookies_key); - downloader.setCookies(prefs.getString(key, "")); + downloader.setCookie(ReCaptchaActivity.RECAPTCHA_COOKIES_KEY, prefs.getString(key, "")); + downloader.updateAgeRestrictedContentCookies(getApplicationContext()); } private void configureRxJavaErrorHandler() { diff --git a/app/src/main/java/org/schabi/newpipe/DownloaderImpl.java b/app/src/main/java/org/schabi/newpipe/DownloaderImpl.java index ed517f160..ac6e9ef59 100644 --- a/app/src/main/java/org/schabi/newpipe/DownloaderImpl.java +++ b/app/src/main/java/org/schabi/newpipe/DownloaderImpl.java @@ -1,15 +1,20 @@ package org.schabi.newpipe; +import android.content.Context; import android.os.Build; -import android.text.TextUtils; +import android.preference.PreferenceManager; import androidx.annotation.NonNull; import androidx.annotation.Nullable; +import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Request; import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import org.schabi.newpipe.util.CookieUtils; +import org.schabi.newpipe.util.InfoCache; +import org.schabi.newpipe.util.ServiceHelper; import org.schabi.newpipe.util.TLSSocketFactoryCompat; import java.io.IOException; @@ -20,6 +25,7 @@ import java.security.KeyStoreException; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -41,8 +47,11 @@ public final class DownloaderImpl extends Downloader { public static final String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:68.0) Gecko/20100101 Firefox/68.0"; + public static final String YOUTUBE_AGE_RESTRICTED_CONTENT_COOKIE_KEY = "youtube_age_restricted_content_cookie_key"; + public static final String YOUTUBE_AGE_RESTRICTED_CONTENT_COOKIE = "PREF=f2=8000000"; + private static DownloaderImpl instance; - private String mCookies; + private Map mCookies; private OkHttpClient client; private DownloaderImpl(final OkHttpClient.Builder builder) { @@ -54,6 +63,7 @@ public final class DownloaderImpl extends Downloader { // .cache(new Cache(new File(context.getExternalCacheDir(), "okhttp"), // 16 * 1024 * 1024)) .build(); + this.mCookies = new HashMap<>(); } /** @@ -122,11 +132,35 @@ public final class DownloaderImpl extends Downloader { } public String getCookies() { - return mCookies; + return CookieUtils.concatCookies(mCookies.values()); } - public void setCookies(final String cookies) { - mCookies = cookies; + public String getCookie(final String key){ + return mCookies.get(key); + } + + public void setCookie(final String key, final String value){ + mCookies.put(key, value); + } + + public void removeCookie(final String key){ + mCookies.remove(key); + } + + public void updateAgeRestrictedContentCookies(Context context){ + String showAgeRestrictedContentKey = context.getString(R.string.show_age_restricted_content); + int currentServiceId = ServiceHelper.getSelectedServiceId(context); + boolean showAgeRestrictedContent = PreferenceManager.getDefaultSharedPreferences(context).getBoolean(showAgeRestrictedContentKey, false); + updateAgeRestrictedContentCookies(currentServiceId, showAgeRestrictedContent); + } + + public void updateAgeRestrictedContentCookies(int currentServiceId, boolean showAgeRestrictedContent) { + if (currentServiceId == ServiceList.YouTube.getServiceId() && !showAgeRestrictedContent) { + setCookie(YOUTUBE_AGE_RESTRICTED_CONTENT_COOKIE_KEY, YOUTUBE_AGE_RESTRICTED_CONTENT_COOKIE); + } else { + removeCookie(YOUTUBE_AGE_RESTRICTED_CONTENT_COOKIE_KEY); + } + InfoCache.getInstance().clearCache(); } /** @@ -152,8 +186,8 @@ public final class DownloaderImpl extends Downloader { .method("GET", null).url(siteUrl) .addHeader("User-Agent", USER_AGENT); - if (!TextUtils.isEmpty(mCookies)) { - requestBuilder.addHeader("Cookie", mCookies); + if (!mCookies.isEmpty()) { + requestBuilder.addHeader("Cookie", getCookies()); } final okhttp3.Request request = requestBuilder.build(); @@ -192,8 +226,8 @@ public final class DownloaderImpl extends Downloader { .method(httpMethod, requestBody).url(url) .addHeader("User-Agent", USER_AGENT); - if (!TextUtils.isEmpty(mCookies)) { - requestBuilder.addHeader("Cookie", mCookies); + if (!mCookies.isEmpty()) { + requestBuilder.addHeader("Cookie", getCookies()); } for (Map.Entry> pair : headers.entrySet()) { diff --git a/app/src/main/java/org/schabi/newpipe/MainActivity.java b/app/src/main/java/org/schabi/newpipe/MainActivity.java index e6269dd5f..b36cdadb4 100644 --- a/app/src/main/java/org/schabi/newpipe/MainActivity.java +++ b/app/src/main/java/org/schabi/newpipe/MainActivity.java @@ -244,6 +244,7 @@ public class MainActivity extends AppCompatActivity { ServiceHelper.setSelectedServiceId(this, item.getItemId()); drawerItems.getMenu().getItem(ServiceHelper.getSelectedServiceId(this)) .setChecked(true); + DownloaderImpl.getInstance().updateAgeRestrictedContentCookies(getApplicationContext()); } private void tabSelected(final MenuItem item) throws ExtractionException { diff --git a/app/src/main/java/org/schabi/newpipe/ReCaptchaActivity.java b/app/src/main/java/org/schabi/newpipe/ReCaptchaActivity.java index 49fb6b179..40ea4fd58 100644 --- a/app/src/main/java/org/schabi/newpipe/ReCaptchaActivity.java +++ b/app/src/main/java/org/schabi/newpipe/ReCaptchaActivity.java @@ -51,6 +51,7 @@ public class ReCaptchaActivity extends AppCompatActivity { public static final String RECAPTCHA_URL_EXTRA = "recaptcha_url_extra"; public static final String TAG = ReCaptchaActivity.class.toString(); public static final String YT_URL = "https://www.youtube.com"; + public static final String RECAPTCHA_COOKIES_KEY = "recaptcha_cookies"; private WebView webView; private String foundCookies = ""; @@ -168,7 +169,7 @@ public class ReCaptchaActivity extends AppCompatActivity { prefs.edit().putString(key, foundCookies).apply(); // give cookies to Downloader class - DownloaderImpl.getInstance().setCookies(foundCookies); + DownloaderImpl.getInstance().setCookie(RECAPTCHA_COOKIES_KEY, foundCookies); setResult(RESULT_OK); } diff --git a/app/src/main/java/org/schabi/newpipe/settings/ContentSettingsFragment.java b/app/src/main/java/org/schabi/newpipe/settings/ContentSettingsFragment.java index bc2765387..1b0f7090c 100644 --- a/app/src/main/java/org/schabi/newpipe/settings/ContentSettingsFragment.java +++ b/app/src/main/java/org/schabi/newpipe/settings/ContentSettingsFragment.java @@ -2,6 +2,7 @@ package org.schabi.newpipe.settings; import android.app.Activity; import android.app.AlertDialog; +import android.content.Context; import android.content.DialogInterface; import android.content.Intent; import android.content.SharedPreferences; @@ -17,6 +18,7 @@ import androidx.preference.Preference; import com.nononsenseapps.filepicker.Utils; import com.nostra13.universalimageloader.core.ImageLoader; +import org.schabi.newpipe.DownloaderImpl; import org.schabi.newpipe.NewPipeDatabase; import org.schabi.newpipe.R; import org.schabi.newpipe.extractor.NewPipe; @@ -56,6 +58,7 @@ public class ContentSettingsFragment extends BasePreferenceFragment { private File newpipeSettings; private String thumbnailLoadToggleKey; + private String showAgeRestrictedContentKey; private Localization initialSelectedLocalization; private ContentCountry initialSelectedContentCountry; @@ -65,6 +68,7 @@ public class ContentSettingsFragment extends BasePreferenceFragment { public void onCreate(@Nullable final Bundle savedInstanceState) { super.onCreate(savedInstanceState); thumbnailLoadToggleKey = getString(R.string.download_thumbnail_key); + showAgeRestrictedContentKey = getString(R.string.show_age_restricted_content); initialSelectedLocalization = org.schabi.newpipe.util.Localization .getPreferredLocalization(requireContext()); @@ -86,6 +90,13 @@ public class ContentSettingsFragment extends BasePreferenceFragment { Toast.LENGTH_SHORT).show(); } + if (preference.getKey().equals(showAgeRestrictedContentKey)) { + Context context = getContext(); + if(context != null){ + DownloaderImpl.getInstance().updateAgeRestrictedContentCookies(context); + } + } + return super.onPreferenceTreeClick(preference); } diff --git a/app/src/main/java/org/schabi/newpipe/util/CookieUtils.java b/app/src/main/java/org/schabi/newpipe/util/CookieUtils.java new file mode 100644 index 000000000..e10b83df6 --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/util/CookieUtils.java @@ -0,0 +1,32 @@ +package org.schabi.newpipe.util; + +import android.content.Context; +import android.preference.PreferenceManager; + +import org.jsoup.helper.StringUtil; +import org.schabi.newpipe.DownloaderImpl; +import org.schabi.newpipe.R; + +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +public class CookieUtils { + private CookieUtils() { + } + + public static String concatCookies(Collection cookieStrings) { + Set cookieSet = new HashSet<>(); + for (String cookies : cookieStrings) { + cookieSet.addAll(splitCookies(cookies)); + } + return StringUtil.join(cookieSet, "; ").trim(); + } + + public static Set splitCookies(String cookies) { + return new HashSet<>(Arrays.asList(cookies.split("; *"))); + } +}