neoris/NewPipe
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #167 from theScrabi/crawlerRefactor

Browse Source 
Crawler refactor
This commit is contained in:
Christian Schabesberger 2016-02-05 14:38:48 +01:00
commit 241414f81b
36 changed files with 1842 additions and 1405 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeSearchEngineTest.java
View File

@ -2,8 +2,10 @@ package org.schabi.newpipe.services.youtube;
import android.test.AndroidTestCase;
import org.schabi.newpipe.VideoPreviewInfo;
import org.schabi.newpipe.services.SearchEngine;
import org.schabi.newpipe.crawler.VideoPreviewInfo;
import org.schabi.newpipe.crawler.SearchEngine;
import org.schabi.newpipe.crawler.services.youtube.YoutubeSearchEngine;
import org.schabi.newpipe.Downloader;
import java.util.ArrayList;
@ -35,8 +37,9 @@ public class YoutubeSearchEngineTest extends AndroidTestCase {
public void setUp() throws Exception{
super.setUp();
SearchEngine engine = new YoutubeSearchEngine();
result = engine.search("https://www.youtube.com/results?search_query=bla", 0, "de");
suggestionReply = engine.suggestionList("hello");
result = engine.search("https://www.youtube.com/results?search_query=bla",
0, "de", new Downloader());
suggestionReply = engine.suggestionList("hello", new Downloader());
}
public void testIfNoErrorOccur() {

60
app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractorDefaultTest.java
View File

@ -1,9 +1,14 @@
package org.schabi.newpipe.services.youtube;
import android.test.AndroidTestCase;
import android.util.Log;
import org.schabi.newpipe.services.VideoInfo;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.crawler.CrawlingException;
import org.schabi.newpipe.crawler.ParsingException;
import org.schabi.newpipe.crawler.services.youtube.YoutubeVideoExtractor;
import org.schabi.newpipe.crawler.VideoInfo;
import java.io.IOException;
/**
* Created by the-scrabi on 30.12.15.
@ -28,67 +33,62 @@ import org.schabi.newpipe.services.VideoInfo;
public class YoutubeVideoExtractorDefaultTest extends AndroidTestCase {
private YoutubeVideoExtractor extractor;
public void setUp() {
extractor = new YoutubeVideoExtractor("https://www.youtube.com/watch?v=FmG385_uUys");
public void setUp() throws IOException, CrawlingException {
extractor = new YoutubeVideoExtractor("https://www.youtube.com/watch?v=FmG385_uUys",
new Downloader());
}
public void testGetErrorCode() {
assertEquals(extractor.getErrorCode(), VideoInfo.NO_ERROR);
}
public void testGetErrorMessage() {
assertEquals(extractor.getErrorMessage(), "");
}
public void testGetTimeStamp() {
public void testGetInvalidTimeStamp() throws ParsingException {
assertTrue(Integer.toString(extractor.getTimeStamp()),
extractor.getTimeStamp() >= 0);
extractor.getTimeStamp() <= 0);
}
public void testGetTitle() {
public void testGetValidTimeStamp() throws CrawlingException, IOException {
YoutubeVideoExtractor extractor =
new YoutubeVideoExtractor("https://youtu.be/FmG385_uUys?t=174", new Downloader());
assertTrue(Integer.toString(extractor.getTimeStamp()),
extractor.getTimeStamp() == 174);
}
public void testGetTitle() throws ParsingException {
assertTrue(!extractor.getTitle().isEmpty());
}
public void testGetDescription() {
public void testGetDescription() throws ParsingException {
assertTrue(extractor.getDescription() != null);
}
public void testGetUploader() {
public void testGetUploader() throws ParsingException {
assertTrue(!extractor.getUploader().isEmpty());
}
public void testGetLength() {
public void testGetLength() throws ParsingException {
assertTrue(extractor.getLength() > 0);
}
public void testGetViews() {
public void testGetViews() throws ParsingException {
assertTrue(extractor.getLength() > 0);
}
public void testGetUploadDate() {
public void testGetUploadDate() throws ParsingException {
assertTrue(extractor.getUploadDate().length() > 0);
}
public void testGetThumbnailUrl() {
public void testGetThumbnailUrl() throws ParsingException {
assertTrue(extractor.getThumbnailUrl(),
extractor.getThumbnailUrl().contains("https://"));
}
public void testGetUploaderThumbnailUrl() {
public void testGetUploaderThumbnailUrl() throws ParsingException {
assertTrue(extractor.getUploaderThumbnailUrl(),
extractor.getUploaderThumbnailUrl().contains("https://"));
}
public void testGetAudioStreams() {
for(VideoInfo.AudioStream s : extractor.getAudioStreams()) {
assertTrue(s.url,
s.url.contains("https://"));
assertTrue(s.bandwidth > 0);
assertTrue(s.samplingRate > 0);
}
public void testGetAudioStreams() throws ParsingException {
assertTrue(extractor.getAudioStreams() == null);
}
public void testGetVideoStreams() {
public void testGetVideoStreams() throws ParsingException {
for(VideoInfo.VideoStream s : extractor.getVideoStreams()) {
assertTrue(s.url,
s.url.contains("https://"));

38
app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractorGemaTest.java
View File

@ -2,7 +2,13 @@ package org.schabi.newpipe.services.youtube;
import android.test.AndroidTestCase;
import org.schabi.newpipe.services.VideoInfo;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.crawler.CrawlingException;
import org.schabi.newpipe.crawler.services.youtube.YoutubeVideoExtractor;
import org.schabi.newpipe.crawler.VideoInfo;
import org.schabi.newpipe.Downloader;
import java.io.IOException;
/**
* Created by the-scrabi on 30.12.15.
@ -31,29 +37,15 @@ public class YoutubeVideoExtractorGemaTest extends AndroidTestCase {
// Deaktivate this Test Case bevore uploading it githup, otherwise CI will fail.
private static final boolean testActive = false;
private YoutubeVideoExtractor extractor;
public void setUp() {
public void testGemaError() throws IOException, CrawlingException {
if(testActive) {
extractor = new YoutubeVideoExtractor("https://www.youtube.com/watch?v=3O1_3zBUKM8");
}
}
public void testGetErrorCode() {
if(testActive) {
assertEquals(extractor.getErrorCode(), VideoInfo.ERROR_BLOCKED_BY_GEMA);
} else {
assertTrue(true);
}
}
public void testGetErrorMessage() {
if(testActive) {
assertTrue(extractor.getErrorMessage(),
extractor.getErrorMessage().contains("GEMA"));
} else {
assertTrue(true);
try {
new YoutubeVideoExtractor("https://www.youtube.com/watch?v=3O1_3zBUKM8",
new Downloader());
assertTrue("Gema exception not thrown", false);
} catch(YoutubeVideoExtractor.GemaException ge) {
assertTrue(true);
}
}
}
}

35
app/src/main/java/org/schabi/newpipe/ActionBarHandler.java
View File

@ -16,13 +16,15 @@ import android.view.MenuInflater;
import android.view.MenuItem;
import android.widget.ArrayAdapter;
import org.schabi.newpipe.services.MediaFormat;
import org.schabi.newpipe.services.VideoInfo;
import org.schabi.newpipe.crawler.MediaFormat;
import org.schabi.newpipe.crawler.VideoInfo;
import java.util.List;
/**
* Created by Christian Schabesberger on 18.08.15.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* DetailsMenuHandler.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
@ -49,7 +51,7 @@ class ActionBarHandler {
private Bitmap videoThumbnail = null;
private String channelName = "";
private AppCompatActivity activity;
private VideoInfo.VideoStream[] videoStreams = null;
private List<VideoInfo.VideoStream> videoStreams = null;
private VideoInfo.AudioStream audioStream = null;
private int selectedStream = -1;
private String videoTitle = "";
@ -93,19 +95,21 @@ class ActionBarHandler {
}
@SuppressWarnings("deprecation")
public void setStreams(VideoInfo.VideoStream[] videoStreams, VideoInfo.AudioStream[] audioStreams) {
public void setStreams(List<VideoInfo.VideoStream> videoStreams,
List<VideoInfo.AudioStream> audioStreams) {
this.videoStreams = videoStreams;
selectedStream = 0;
defaultPreferences = PreferenceManager.getDefaultSharedPreferences(activity);
String[] itemArray = new String[videoStreams.length];
String[] itemArray = new String[videoStreams.size()];
String defaultResolution = defaultPreferences
.getString(activity.getString(R.string.default_resolution_key),
activity.getString(R.string.default_resolution_value));
int defaultResolutionPos = 0;
for(int i = 0; i < videoStreams.length; i++) {
itemArray[i] = MediaFormat.getNameById(videoStreams[i].format) + " " + videoStreams[i].resolution;
if(defaultResolution.equals(videoStreams[i].resolution)) {
for(int i = 0; i < videoStreams.size(); i++) {
VideoInfo.VideoStream item = videoStreams.get(i);
itemArray[i] = MediaFormat.getNameById(item.format) + " " + item.resolution;
if(defaultResolution.equals(item.resolution)) {
defaultResolutionPos = i;
}
}
@ -209,6 +213,8 @@ class ActionBarHandler {
public void playVideo() {
// ----------- THE MAGIC MOMENT ---------------
if(!videoTitle.isEmpty()) {
VideoInfo.VideoStream selectedStreamItem = videoStreams.get(selectedStream);
if (PreferenceManager.getDefaultSharedPreferences(activity)
.getBoolean(activity.getString(R.string.use_external_video_player_key), false)) {
@ -217,8 +223,8 @@ class ActionBarHandler {
try {
intent.setAction(Intent.ACTION_VIEW);
intent.setDataAndType(Uri.parse(videoStreams[selectedStream].url),
MediaFormat.getMimeById(videoStreams[selectedStream].format));
intent.setDataAndType(Uri.parse(selectedStreamItem.url),
MediaFormat.getMimeById(selectedStreamItem.format));
intent.putExtra(Intent.EXTRA_TITLE, videoTitle);
intent.putExtra("title", videoTitle);
@ -248,7 +254,7 @@ class ActionBarHandler {
// Internal Player
Intent intent = new Intent(activity, PlayVideoActivity.class);
intent.putExtra(PlayVideoActivity.VIDEO_TITLE, videoTitle);
intent.putExtra(PlayVideoActivity.STREAM_URL, videoStreams[selectedStream].url);
intent.putExtra(PlayVideoActivity.STREAM_URL, selectedStreamItem.url);
intent.putExtra(PlayVideoActivity.VIDEO_URL, websiteUrl);
intent.putExtra(PlayVideoActivity.START_POSITION, startPosition);
activity.startActivity(intent); //also HERE !!!
@ -264,13 +270,14 @@ class ActionBarHandler {
private void downloadVideo() {
if(!videoTitle.isEmpty()) {
String videoSuffix = "." + MediaFormat.getSuffixById(videoStreams[selectedStream].format);
VideoInfo.VideoStream selectedStreamItem = videoStreams.get(selectedStream);
String videoSuffix = "." + MediaFormat.getSuffixById(selectedStreamItem.format);
String audioSuffix = "." + MediaFormat.getSuffixById(audioStream.format);
Bundle args = new Bundle();
args.putString(DownloadDialog.FILE_SUFFIX_VIDEO, videoSuffix);
args.putString(DownloadDialog.FILE_SUFFIX_AUDIO, audioSuffix);
args.putString(DownloadDialog.TITLE, videoTitle);
args.putString(DownloadDialog.VIDEO_URL, videoStreams[selectedStream].url);
args.putString(DownloadDialog.VIDEO_URL, selectedStreamItem.url);
args.putString(DownloadDialog.AUDIO_URL, audioStream.url);
DownloadDialog downloadDialog = new DownloadDialog();
downloadDialog.setArguments(args);

2
app/src/main/java/org/schabi/newpipe/DownloadDialog.java
View File

@ -107,7 +107,7 @@ public class DownloadDialog extends DialogFragment {
long id = 0;
if (App.isUsingTor()) {
// if using Tor, do not use DownloadManager because the proxy cannot be set
Downloader.downloadFile(getContext(), url, saveFilePath, title);
FileDownloader.downloadFile(getContext(), url, saveFilePath, title);
} else {
DownloadManager dm = (DownloadManager) context.getSystemService(Context.DOWNLOAD_SERVICE);
DownloadManager.Request request = new DownloadManager.Request(

203
app/src/main/java/org/schabi/newpipe/Downloader.java
View File

@ -1,24 +1,8 @@
package org.schabi.newpipe;
import android.app.NotificationManager;
import android.content.Context;
import android.content.SharedPreferences;
import android.graphics.drawable.BitmapDrawable;
import android.graphics.drawable.Drawable;
import android.os.AsyncTask;
import android.preference.PreferenceManager;
import android.support.v4.app.NotificationCompat;
import android.util.Log;
import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.UnknownHostException;
@ -27,9 +11,9 @@ import javax.net.ssl.HttpsURLConnection;
import info.guardianproject.netcipher.NetCipher;
/**
* Created by Christian Schabesberger on 14.08.15.
* Created by Christian Schabesberger on 28.01.16.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* Downloader.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
@ -46,190 +30,61 @@ import info.guardianproject.netcipher.NetCipher;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class Downloader extends AsyncTask<Void, Integer, Void> {
public static final String TAG = "Downloader";
public class Downloader implements org.schabi.newpipe.crawler.Downloader {
private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0";
private NotificationManager nm;
private NotificationCompat.Builder builder;
private int notifyId = 0x1234;
private int fileSize = 0xffffffff;
private final Context context;
private final String fileURL;
private final File saveFilePath;
private final String title;
private final String debugContext;
public Downloader(Context context, String fileURL, File saveFilePath, String title) {
this.context = context;
this.fileURL = fileURL;
this.saveFilePath = saveFilePath;
this.title = title;
this.debugContext = "'" + fileURL +
"' => '" + saveFilePath + "'";
}
/**Download the text file at the supplied URL as in download(String),
* but set the HTTP header field "Accept-Language" to the supplied string.
* @param siteUrl the URL of the text file to return the contents of
* @param language the language (usually a 2-character code) to set as the preferred language
* @return the contents of the specified text file*/
public static String download(String siteUrl, String language) {
String ret = "";
try {
URL url = new URL(siteUrl);
//HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
HttpsURLConnection con = NetCipher.getHttpsURLConnection(url);
con.setRequestProperty("Accept-Language", language);
ret = dl(con);
}
catch(Exception e) {
e.printStackTrace();
}
return ret;
public String download(String siteUrl, String language) throws IOException {
URL url = new URL(siteUrl);
//HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
HttpsURLConnection con = NetCipher.getHttpsURLConnection(url);
con.setRequestProperty("Accept-Language", language);
return dl(con);
}
/**Common functionality between download(String url) and download(String url, String language)*/
private static String dl(HttpsURLConnection con) throws IOException {
StringBuilder response = new StringBuilder();
BufferedReader in = null;
try {
con.setRequestMethod("GET");
con.setRequestProperty("User-Agent", USER_AGENT);
BufferedReader in = new BufferedReader(
in = new BufferedReader(
new InputStreamReader(con.getInputStream()));
String inputLine;
while((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
in.close();
}
catch(UnknownHostException uhe) {//thrown when there's no internet connection
uhe.printStackTrace();
} catch(UnknownHostException uhe) {//thrown when there's no internet connection
throw new IOException("unknown host or no network", uhe);
//Toast.makeText(getActivity(), uhe.getMessage(), Toast.LENGTH_LONG).show();
} catch(Exception e) {
throw new IOException(e);
} finally {
if(in != null) {
in.close();
}
}
return response.toString();
}
/**Download (via HTTP) the text file located at the supplied URL, and return its contents.
* Primarily intended for downloading web pages.
* @param siteUrl the URL of the text file to download
* @return the contents of the specified text file*/
public static String download(String siteUrl) {
String ret = "";
try {
URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
//HttpsURLConnection con = NetCipher.getHttpsURLConnection(url);
ret = dl(con);
}
catch(Exception e) {
e.printStackTrace();
}
return ret;
/**Download (via HTTP) the text file located at the supplied URL, and return its contents.
* Primarily intended for downloading web pages.
* @param siteUrl the URL of the text file to download
* @return the contents of the specified text file*/
public String download(String siteUrl) throws IOException {
URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
//HttpsURLConnection con = NetCipher.getHttpsURLConnection(url);
return dl(con);
}
/**
* Downloads a file from a URL in the background using an {@link AsyncTask}.
*
* @param fileURL HTTP URL of the file to be downloaded
* @param saveFilePath path of the directory to save the file
* @param title
* @throws IOException
*/
public static void downloadFile(final Context context, final String fileURL, final File saveFilePath, String title) {
new Downloader(context, fileURL, saveFilePath, title).execute();
}
/** AsyncTask impl: executed in gui thread */
@Override
protected void onPreExecute() {
super.onPreExecute();
nm = (NotificationManager) context.getSystemService(Context.NOTIFICATION_SERVICE);
Drawable icon = context.getResources().getDrawable(R.mipmap.ic_launcher);
builder = new NotificationCompat.Builder(context)
.setSmallIcon(android.R.drawable.stat_sys_download)
.setLargeIcon(((BitmapDrawable) icon).getBitmap())
.setContentTitle(saveFilePath.getName())
.setContentText(saveFilePath.getAbsolutePath())
.setProgress(fileSize, 0, false);
nm.notify(notifyId, builder.build());
}
/** AsyncTask impl: executed in background thread does the download */
@Override
protected Void doInBackground(Void... voids) {
HttpsURLConnection con = null;
InputStream inputStream = null;
FileOutputStream outputStream = null;
try {
con = NetCipher.getHttpsURLConnection(fileURL);
int responseCode = con.getResponseCode();
// always check HTTP response code first
if (responseCode == HttpURLConnection.HTTP_OK) {
fileSize = con.getContentLength();
inputStream = new BufferedInputStream(con.getInputStream());
outputStream = new FileOutputStream(saveFilePath);
int bufferSize = 8192;
int downloaded = 0;
int bytesRead = -1;
byte[] buffer = new byte[bufferSize];
while ((bytesRead = inputStream.read(buffer)) != -1) {
outputStream.write(buffer, 0, bytesRead);
downloaded += bytesRead;
if (downloaded % 50000 < bufferSize) {
publishProgress(downloaded);
}
}
publishProgress(bufferSize);
} else {
Log.i(TAG, "No file to download. Server replied HTTP code: " + responseCode);
}
} catch (IOException e) {
Log.e(TAG, "No file to download. Server replied HTTP code: ", e);
e.printStackTrace();
} finally {
try {
if (outputStream != null) {
outputStream.close();
}
if (inputStream != null) {
inputStream.close();
}
} catch (IOException e) {
e.printStackTrace();
}
if (con != null) {
con.disconnect();
}
}
return null;
}
@Override
protected void onProgressUpdate(Integer... progress) {
builder.setProgress(fileSize, progress[0], false);
nm.notify(notifyId, builder.build());
}
@Override
protected void onPostExecute(Void aVoid) {
super.onPostExecute(aVoid);
nm.cancel(notifyId);
}
}

169
app/src/main/java/org/schabi/newpipe/FileDownloader.java Normal file
View File

@ -0,0 +1,169 @@
package org.schabi.newpipe;
import android.app.NotificationManager;
import android.content.Context;
import android.content.SharedPreferences;
import android.graphics.drawable.BitmapDrawable;
import android.graphics.drawable.Drawable;
import android.os.AsyncTask;
import android.preference.PreferenceManager;
import android.support.v4.app.NotificationCompat;
import android.util.Log;
import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.UnknownHostException;
import javax.net.ssl.HttpsURLConnection;
import info.guardianproject.netcipher.NetCipher;
/**
* Created by Christian Schabesberger on 14.08.15.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
* FileDownloader.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class FileDownloader extends AsyncTask<Void, Integer, Void> {
public static final String TAG = "FileDownloader";
private NotificationManager nm;
private NotificationCompat.Builder builder;
private int notifyId = 0x1234;
private int fileSize = 0xffffffff;
private final Context context;
private final String fileURL;
private final File saveFilePath;
private final String title;
private final String debugContext;
public FileDownloader(Context context, String fileURL, File saveFilePath, String title) {
this.context = context;
this.fileURL = fileURL;
this.saveFilePath = saveFilePath;
this.title = title;
this.debugContext = "'" + fileURL +
"' => '" + saveFilePath + "'";
}
/**
* Downloads a file from a URL in the background using an {@link AsyncTask}.
*
* @param fileURL HTTP URL of the file to be downloaded
* @param saveFilePath path of the directory to save the file
* @param title
* @throws IOException
*/
public static void downloadFile(final Context context, final String fileURL, final File saveFilePath, String title) {
new FileDownloader(context, fileURL, saveFilePath, title).execute();
}
/** AsyncTask impl: executed in gui thread */
@Override
protected void onPreExecute() {
super.onPreExecute();
nm = (NotificationManager) context.getSystemService(Context.NOTIFICATION_SERVICE);
Drawable icon = context.getResources().getDrawable(R.mipmap.ic_launcher);
builder = new NotificationCompat.Builder(context)
.setSmallIcon(android.R.drawable.stat_sys_download)
.setLargeIcon(((BitmapDrawable) icon).getBitmap())
.setContentTitle(saveFilePath.getName())
.setContentText(saveFilePath.getAbsolutePath())
.setProgress(fileSize, 0, false);
nm.notify(notifyId, builder.build());
}
/** AsyncTask impl: executed in background thread does the download */
@Override
protected Void doInBackground(Void... voids) {
HttpsURLConnection con = null;
InputStream inputStream = null;
FileOutputStream outputStream = null;
try {
con = NetCipher.getHttpsURLConnection(fileURL);
int responseCode = con.getResponseCode();
// always check HTTP response code first
if (responseCode == HttpURLConnection.HTTP_OK) {
fileSize = con.getContentLength();
inputStream = new BufferedInputStream(con.getInputStream());
outputStream = new FileOutputStream(saveFilePath);
int bufferSize = 8192;
int downloaded = 0;
int bytesRead = -1;
byte[] buffer = new byte[bufferSize];
while ((bytesRead = inputStream.read(buffer)) != -1) {
outputStream.write(buffer, 0, bytesRead);
downloaded += bytesRead;
if (downloaded % 50000 < bufferSize) {
publishProgress(downloaded);
}
}
publishProgress(bufferSize);
} else {
Log.i(TAG, "No file to download. Server replied HTTP code: " + responseCode);
}
} catch (IOException e) {
Log.e(TAG, "No file to download. Server replied HTTP code: ", e);
e.printStackTrace();
} finally {
try {
if (outputStream != null) {
outputStream.close();
}
if (inputStream != null) {
inputStream.close();
}
} catch (IOException e) {
e.printStackTrace();
}
if (con != null) {
con.disconnect();
}
}
return null;
}
@Override
protected void onProgressUpdate(Integer... progress) {
builder.setProgress(fileSize, progress[0], false);
nm.notify(notifyId, builder.build());
}
@Override
protected void onPostExecute(Void aVoid) {
super.onPostExecute(aVoid);
nm.cancel(notifyId);
}
}

2
app/src/main/java/org/schabi/newpipe/VideoInfoItemViewCreator.java
View File

@ -7,6 +7,8 @@ import android.view.ViewGroup;
import android.widget.ImageView;
import android.widget.TextView;
import org.schabi.newpipe.crawler.VideoPreviewInfo;
/**
* Created by Christian Schabesberger on 24.10.15.
*

6
app/src/main/java/org/schabi/newpipe/VideoItemDetailActivity.java
View File

@ -11,8 +11,8 @@ import android.view.Menu;
import android.view.MenuItem;
import android.widget.Toast;
import org.schabi.newpipe.services.ServiceList;
import org.schabi.newpipe.services.StreamingService;
import org.schabi.newpipe.crawler.ServiceList;
import org.schabi.newpipe.crawler.StreamingService;
/**
@ -73,7 +73,7 @@ public class VideoItemDetailActivity extends AppCompatActivity {
StreamingService[] serviceList = ServiceList.getServices();
//VideoExtractor videoExtractor = null;
for (int i = 0; i < serviceList.length; i++) {
if (serviceList[i].acceptUrl(videoUrl)) {
if (serviceList[i].getUrlIdHandler().acceptUrl(videoUrl)) {
arguments.putInt(VideoItemDetailFragment.STREAMING_SERVICE, i);
currentStreamingService = i;
//videoExtractor = ServiceList.getService(i).getExtractorInstance();

268
app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java
View File

@ -16,7 +16,6 @@ import android.support.v4.app.Fragment;
import android.support.v7.app.AppCompatActivity;
import android.text.Html;
import android.text.method.LinkMovementMethod;
import android.util.DisplayMetrics;
import android.util.Log;
import android.view.LayoutInflater;
import android.view.Menu;
@ -32,14 +31,20 @@ import android.widget.TextView;
import android.view.MenuItem;
import android.widget.Toast;
import java.io.IOException;
import java.net.URL;
import java.nio.charset.MalformedInputException;
import java.util.ArrayList;
import java.util.Vector;
import org.schabi.newpipe.services.VideoExtractor;
import org.schabi.newpipe.services.ServiceList;
import org.schabi.newpipe.services.StreamingService;
import org.schabi.newpipe.services.VideoInfo;
import org.schabi.newpipe.crawler.CrawlingException;
import org.schabi.newpipe.crawler.ParsingException;
import org.schabi.newpipe.crawler.VideoPreviewInfo;
import org.schabi.newpipe.crawler.VideoExtractor;
import org.schabi.newpipe.crawler.ServiceList;
import org.schabi.newpipe.crawler.StreamingService;
import org.schabi.newpipe.crawler.VideoInfo;
import org.schabi.newpipe.crawler.services.youtube.YoutubeVideoExtractor;
/**
@ -68,7 +73,6 @@ public class VideoItemDetailFragment extends Fragment {
* The fragment argument representing the item ID that this fragment
* represents.
*/
//public static final String ARG_ITEM_ID = "item_id";
public static final String VIDEO_URL = "video_url";
public static final String STREAMING_SERVICE = "streaming_service";
public static final String AUTO_PLAY = "auto_play";
@ -87,7 +91,6 @@ public class VideoItemDetailFragment extends Fragment {
private FloatingActionButton playVideoButton;
private final Point initialThumbnailPos = new Point(0, 0);
public interface OnInvokeCreateOptionsMenuListener {
void createOptionsMenu();
}
@ -108,45 +111,65 @@ public class VideoItemDetailFragment extends Fragment {
@Override
public void run() {
try {
this.videoExtractor = service.getExtractorInstance(videoUrl);
VideoInfo videoInfo = videoExtractor.getVideoInfo();
videoExtractor = service.getExtractorInstance(videoUrl, new Downloader());
VideoInfo videoInfo = VideoInfo.getVideoInfo(videoExtractor, new Downloader());
h.post(new VideoResultReturnedRunnable(videoInfo));
if (videoInfo.errorCode == VideoInfo.NO_ERROR) {
h.post(new SetThumbnailRunnable(
//todo: make bitmaps not bypass tor
BitmapFactory.decodeStream(
new URL(videoInfo.thumbnail_url)
.openConnection()
.getInputStream()),
SetThumbnailRunnable.VIDEO_THUMBNAIL));
h.post(new SetThumbnailRunnable(
BitmapFactory.decodeStream(
new URL(videoInfo.uploader_thumbnail_url)
.openConnection()
.getInputStream()),
SetThumbnailRunnable.CHANNEL_THUMBNAIL));
if (showNextVideoItem) {
h.post(new SetThumbnailRunnable(
BitmapFactory.decodeStream(
new URL(videoInfo.thumbnail_url)
new URL(videoInfo.nextVideo.thumbnail_url)
.openConnection()
.getInputStream()),
SetThumbnailRunnable.VIDEO_THUMBNAIL));
h.post(new SetThumbnailRunnable(
BitmapFactory.decodeStream(
new URL(videoInfo.uploader_thumbnail_url)
.openConnection()
.getInputStream()),
SetThumbnailRunnable.CHANNEL_THUMBNAIL));
if(showNextVideoItem) {
h.post(new SetThumbnailRunnable(
BitmapFactory.decodeStream(
new URL(videoInfo.nextVideo.thumbnail_url)
.openConnection()
.getInputStream()),
SetThumbnailRunnable.NEXT_VIDEO_THUMBNAIL));
}
SetThumbnailRunnable.NEXT_VIDEO_THUMBNAIL));
}
} catch (Exception e) {
} catch (MalformedInputException e) {
postNewErrorToast(h, R.string.could_not_load_thumbnails);
e.printStackTrace();
} catch (IOException e) {
postNewErrorToast(h, R.string.network_error);
e.printStackTrace();
}
// custom service related exceptions
catch (YoutubeVideoExtractor.DecryptException de) {
postNewErrorToast(h, R.string.youtube_signature_decryption_error);
de.printStackTrace();
} catch (YoutubeVideoExtractor.GemaException ge) {
h.post(new Runnable() {
@Override
public void run() {
progressBar.setVisibility(View.GONE);
// This is poor style, but unless we have better error handling in the
// crawler, this may not be better.
Toast.makeText(VideoItemDetailFragment.this.getActivity(),
R.string.network_error, Toast.LENGTH_LONG).show();
onErrorBlockedByGema();
}
});
}
// ----------------------------------------
catch(VideoExtractor.ContentNotAvailableException e) {
h.post(new Runnable() {
@Override
public void run() {
onNotSpecifiedContentError();
}
});
e.printStackTrace();
} catch (ParsingException e) {
postNewErrorToast(h, e.getMessage());
e.printStackTrace();
} catch(Exception e) {
postNewErrorToast(h, R.string.general_error);
e.printStackTrace();
}
}
}
@ -213,7 +236,7 @@ public class VideoItemDetailFragment extends Fragment {
private void updateInfo(VideoInfo info) {
currentVideoInfo = info;
Resources res = activity.getResources();
try {
VideoInfoItemViewCreator videoItemViewCreator =
new VideoInfoItemViewCreator(LayoutInflater.from(getActivity()));
@ -226,107 +249,77 @@ public class VideoItemDetailFragment extends Fragment {
TextView thumbsDownView = (TextView) activity.findViewById(R.id.detailThumbsDownCountView);
TextView uploadDateView = (TextView) activity.findViewById(R.id.detailUploadDateView);
TextView descriptionView = (TextView) activity.findViewById(R.id.detailDescriptionView);
ImageView thumbnailView = (ImageView) activity.findViewById(R.id.detailThumbnailView);
FrameLayout nextVideoFrame = (FrameLayout) activity.findViewById(R.id.detailNextVideoFrame);
RelativeLayout nextVideoRootFrame =
(RelativeLayout) activity.findViewById(R.id.detailNextVideoRootLayout);
Button backgroundButton = (Button)
activity.findViewById(R.id.detailVideoThumbnailWindowBackgroundButton);
progressBar.setVisibility(View.GONE);
switch (info.errorCode) {
case VideoInfo.NO_ERROR: {
View nextVideoView = videoItemViewCreator
.getViewFromVideoInfoItem(null, nextVideoFrame, info.nextVideo, getContext());
nextVideoFrame.addView(nextVideoView);
View nextVideoView = videoItemViewCreator
.getViewFromVideoInfoItem(null, nextVideoFrame, info.nextVideo, getContext());
nextVideoFrame.addView(nextVideoView);
Button nextVideoButton = (Button) activity.findViewById(R.id.detailNextVideoButton);
Button similarVideosButton = (Button) activity.findViewById(R.id.detailShowSimilarButton);
Button nextVideoButton = (Button) activity.findViewById(R.id.detailNextVideoButton);
Button similarVideosButton = (Button) activity.findViewById(R.id.detailShowSimilarButton);
textContentLayout.setVisibility(View.VISIBLE);
playVideoButton.setVisibility(View.VISIBLE);
if (!showNextVideoItem) {
nextVideoRootFrame.setVisibility(View.GONE);
similarVideosButton.setVisibility(View.GONE);
}
textContentLayout.setVisibility(View.VISIBLE);
playVideoButton.setVisibility(View.VISIBLE);
if (!showNextVideoItem) {
nextVideoRootFrame.setVisibility(View.GONE);
similarVideosButton.setVisibility(View.GONE);
}
videoTitleView.setText(info.title);
uploaderView.setText(info.uploader);
actionBarHandler.setChannelName(info.uploader);
videoTitleView.setText(info.title);
uploaderView.setText(info.uploader);
actionBarHandler.setChannelName(info.uploader);
String localizedViewCount = Localization.localizeViewCount(info.view_count, getContext());
viewCountView.setText(localizedViewCount);
String localizedViewCount = Localization.localizeViewCount(info.view_count, getContext());
viewCountView.setText(localizedViewCount);
String localizedLikeCount = Localization.localizeNumber(info.like_count, getContext());
thumbsUpView.setText(localizedLikeCount);
String localizedLikeCount = Localization.localizeNumber(info.like_count, getContext());
thumbsUpView.setText(localizedLikeCount);
String localizedDislikeCount = Localization.localizeNumber(info.dislike_count, getContext());
thumbsDownView.setText(localizedDislikeCount);
String localizedDislikeCount = Localization.localizeNumber(info.dislike_count, getContext());
thumbsDownView.setText(localizedDislikeCount);
String localizedDate = Localization.localizeDate(info.upload_date, getContext());
uploadDateView.setText(localizedDate);
String localizedDate = Localization.localizeDate(info.upload_date, getContext());
uploadDateView.setText(localizedDate);
descriptionView.setText(Html.fromHtml(info.description));
descriptionView.setText(Html.fromHtml(info.description));
descriptionView.setMovementMethod(LinkMovementMethod.getInstance());
descriptionView.setMovementMethod(LinkMovementMethod.getInstance());
actionBarHandler.setServiceId(streamingServiceId);
actionBarHandler.setVideoInfo(info.webpage_url, info.title);
actionBarHandler.setStartPosition(info.startPosition);
actionBarHandler.setServiceId(streamingServiceId);
actionBarHandler.setVideoInfo(info.webpage_url, info.title);
actionBarHandler.setStartPosition(info.startPosition);
// parse streams
Vector<VideoInfo.VideoStream> streamsToUse = new Vector<>();
for (VideoInfo.VideoStream i : info.videoStreams) {
if (useStream(i, streamsToUse)) {
streamsToUse.add(i);
}
}
VideoInfo.VideoStream[] streamList = new VideoInfo.VideoStream[streamsToUse.size()];
for (int i = 0; i < streamList.length; i++) {
streamList[i] = streamsToUse.get(i);
}
actionBarHandler.setStreams(streamList, info.audioStreams);
// parse streams
Vector<VideoInfo.VideoStream> streamsToUse = new Vector<>();
for (VideoInfo.VideoStream i : info.videoStreams) {
if (useStream(i, streamsToUse)) {
streamsToUse.add(i);
}
}
nextVideoButton.setOnClickListener(new View.OnClickListener() {
@Override
public void onClick(View v) {
Intent detailIntent =
new Intent(getActivity(), VideoItemDetailActivity.class);
actionBarHandler.setStreams(streamsToUse, info.audioStreams);
nextVideoButton.setOnClickListener(new View.OnClickListener() {
@Override
public void onClick(View v) {
Intent detailIntent =
new Intent(getActivity(), VideoItemDetailActivity.class);
/*detailIntent.putExtra(
VideoItemDetailFragment.ARG_ITEM_ID, currentVideoInfo.nextVideo.id); */
detailIntent.putExtra(
VideoItemDetailFragment.VIDEO_URL, currentVideoInfo.nextVideo.webpage_url);
detailIntent.putExtra(
VideoItemDetailFragment.VIDEO_URL, currentVideoInfo.nextVideo.webpage_url);
detailIntent.putExtra(VideoItemDetailFragment.STREAMING_SERVICE, streamingServiceId);
startActivity(detailIntent);
}
});
detailIntent.putExtra(VideoItemDetailFragment.STREAMING_SERVICE, streamingServiceId);
startActivity(detailIntent);
}
break;
case VideoInfo.ERROR_BLOCKED_BY_GEMA:
thumbnailView.setImageBitmap(BitmapFactory.decodeResource(
getResources(), R.drawable.gruese_die_gema));
backgroundButton.setOnClickListener(new View.OnClickListener() {
@Override
public void onClick(View v) {
Intent intent = new Intent();
intent.setAction(Intent.ACTION_VIEW);
intent.setData(Uri.parse(activity.getString(R.string.c3s_url)));
activity.startActivity(intent);
}
});
break;
case VideoInfo.ERROR_NO_SPECIFIED_ERROR:
thumbnailView.setImageBitmap(BitmapFactory.decodeResource(
getResources(), R.drawable.not_available_monkey));
Toast.makeText(activity, info.errorMessage, Toast.LENGTH_LONG)
.show();
break;
default:
Log.e(TAG, "Video Available Status not known.");
}
});
if(autoPlayEnabled) {
actionBarHandler.playVideo();
@ -337,6 +330,37 @@ public class VideoItemDetailFragment extends Fragment {
}
}
private void onErrorBlockedByGema() {
Button backgroundButton = (Button)
activity.findViewById(R.id.detailVideoThumbnailWindowBackgroundButton);
ImageView thumbnailView = (ImageView) activity.findViewById(R.id.detailThumbnailView);
progressBar.setVisibility(View.GONE);
thumbnailView.setImageBitmap(BitmapFactory.decodeResource(
getResources(), R.drawable.gruese_die_gema));
backgroundButton.setOnClickListener(new View.OnClickListener() {
@Override
public void onClick(View v) {
Intent intent = new Intent();
intent.setAction(Intent.ACTION_VIEW);
intent.setData(Uri.parse(activity.getString(R.string.c3s_url)));
activity.startActivity(intent);
}
});
Toast.makeText(VideoItemDetailFragment.this.getActivity(),
R.string.blocked_by_gema, Toast.LENGTH_LONG).show();
}
private void onNotSpecifiedContentError() {
ImageView thumbnailView = (ImageView) activity.findViewById(R.id.detailThumbnailView);
progressBar.setVisibility(View.GONE);
thumbnailView.setImageBitmap(BitmapFactory.decodeResource(
getResources(), R.drawable.not_available_monkey));
Toast.makeText(activity, R.string.content_not_available, Toast.LENGTH_LONG)
.show();
}
private boolean useStream(VideoInfo.VideoStream stream, Vector<VideoInfo.VideoStream> streams) {
for(VideoInfo.VideoStream i : streams) {
if(i.resolution.equals(stream.resolution)) {
@ -465,4 +489,24 @@ public class VideoItemDetailFragment extends Fragment {
public void setOnInvokeCreateOptionsMenuListener(OnInvokeCreateOptionsMenuListener listener) {
this.onInvokeCreateOptionsMenuListener = listener;
}
private void postNewErrorToast(Handler h, final int stringResource) {
h.post(new Runnable() {
@Override
public void run() {
Toast.makeText(VideoItemDetailFragment.this.getActivity(),
stringResource, Toast.LENGTH_LONG).show();
}
});
}
private void postNewErrorToast(Handler h, final String message) {
h.post(new Runnable() {
@Override
public void run() {
Toast.makeText(VideoItemDetailFragment.this.getActivity(),
message, Toast.LENGTH_LONG).show();
}
});
}
}

3
app/src/main/java/org/schabi/newpipe/VideoItemListActivity.java
View File

@ -17,7 +17,8 @@ import android.view.inputmethod.InputMethodManager;
import java.util.ArrayList;
import org.schabi.newpipe.services.ServiceList;
import org.schabi.newpipe.crawler.VideoPreviewInfo;
import org.schabi.newpipe.crawler.ServiceList;
/**
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>

39
app/src/main/java/org/schabi/newpipe/VideoItemListFragment.java
View File

@ -15,12 +15,15 @@ import android.widget.AbsListView;
import android.widget.ListView;
import android.widget.Toast;
import java.io.IOException;
import java.net.URL;
import java.util.List;
import java.util.Vector;
import org.schabi.newpipe.services.SearchEngine;
import org.schabi.newpipe.services.StreamingService;
import org.schabi.newpipe.crawler.CrawlingException;
import org.schabi.newpipe.crawler.VideoPreviewInfo;
import org.schabi.newpipe.crawler.SearchEngine;
import org.schabi.newpipe.crawler.StreamingService;
/**
@ -108,23 +111,22 @@ public class VideoItemListFragment extends ListFragment {
String searchLanguageKey = getContext().getString(R.string.search_language_key);
String searchLanguage = sp.getString(searchLanguageKey,
getString(R.string.default_language_value));
SearchEngine.Result result = engine.search(query, page, searchLanguage);
SearchEngine.Result result = engine.search(query, page, searchLanguage,
new Downloader());
Log.i(TAG, "language code passed:\""+searchLanguage+"\"");
if(runs) {
h.post(new ResultRunnable(result, requestId));
}
} catch(Exception e) {
} catch(IOException e) {
postNewErrorToast(h, R.string.network_error);
e.printStackTrace();
} catch(CrawlingException ce) {
postNewErrorToast(h, R.string.parsing_error);
ce.printStackTrace();
} catch(Exception e) {
postNewErrorToast(h, R.string.general_error);
e.printStackTrace();
h.post(new Runnable() {
@Override
public void run() {
setListShown(true);
Toast.makeText(getActivity(), getString(R.string.network_error),
Toast.LENGTH_SHORT).show();
}
});
}
}
}
@ -155,6 +157,7 @@ public class VideoItemListFragment extends ListFragment {
if(!downloadedList.get(i)) {
Bitmap thumbnail;
try {
//todo: make bitmaps not bypass tor
thumbnail = BitmapFactory.decodeStream(
new URL(thumbnailUrlList.get(i)).openConnection().getInputStream());
h.post(new SetThumbnailRunnable(i, thumbnail, requestId));
@ -384,4 +387,14 @@ public class VideoItemListFragment extends ListFragment {
mActivatedPosition = position;
}
private void postNewErrorToast(Handler h, final int stringResource) {
h.post(new Runnable() {
@Override
public void run() {
setListShown(true);
Toast.makeText(getActivity(), getString(R.string.network_error),
Toast.LENGTH_SHORT).show();
}
});
}
}

2
app/src/main/java/org/schabi/newpipe/VideoListAdapter.java
View File

@ -9,6 +9,8 @@ import android.view.ViewGroup;
import android.widget.BaseAdapter;
import android.widget.ListView;
import org.schabi.newpipe.crawler.VideoPreviewInfo;
import java.util.List;
import java.util.Vector;

33
app/src/main/java/org/schabi/newpipe/crawler/AbstractVideoInfo.java Normal file
View File

@ -0,0 +1,33 @@
package org.schabi.newpipe.crawler;
import android.graphics.Bitmap;
/**
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
* AbstractVideoInfo.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
/**Common properties between VideoInfo and VideoPreviewInfo.*/
public abstract class AbstractVideoInfo {
public String id = "";
public String title = "";
public String uploader = "";
public String thumbnail_url = "";
public Bitmap thumbnail = null;
public String webpage_url = "";
public String upload_date = "";
public long view_count = -1;
}

37
app/src/main/java/org/schabi/newpipe/crawler/CrawlingException.java Normal file
View File

@ -0,0 +1,37 @@
package org.schabi.newpipe.crawler;
/**
* Created by Christian Schabesberger on 30.01.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* CrawlingException.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class CrawlingException extends Exception {
public CrawlingException() {}
public CrawlingException(String message) {
super(message);
}
public CrawlingException(Throwable cause) {
super(cause);
}
public CrawlingException(String message, Throwable cause) {
super(message, cause);
}
}

102
app/src/main/java/org/schabi/newpipe/crawler/DashMpdParser.java Normal file
View File

@ -0,0 +1,102 @@
package org.schabi.newpipe.crawler;
import android.util.Xml;
import org.xmlpull.v1.XmlPullParser;
import java.io.IOException;
import java.io.StringReader;
import java.util.List;
import java.util.Vector;
/**
* Created by Christian Schabesberger on 02.02.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* DashMpdParser.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class DashMpdParser {
static class DashMpdParsingException extends ParsingException {
DashMpdParsingException(String message, Exception e) {
super(message, e);
}
}
public static List<VideoInfo.AudioStream> getAudioStreams(String dashManifestUrl,
Downloader downloader)
throws DashMpdParsingException {
String dashDoc;
try {
dashDoc = downloader.download(dashManifestUrl);
} catch(IOException ioe) {
throw new DashMpdParsingException("Could not get dash mpd: " + dashManifestUrl, ioe);
}
Vector<VideoInfo.AudioStream> audioStreams = new Vector<>();
try {
XmlPullParser parser = Xml.newPullParser();
parser.setInput(new StringReader(dashDoc));
String tagName = "";
String currentMimeType = "";
int currentBandwidth = -1;
int currentSamplingRate = -1;
boolean currentTagIsBaseUrl = false;
for(int eventType = parser.getEventType();
eventType != XmlPullParser.END_DOCUMENT;
eventType = parser.next() ) {
switch(eventType) {
case XmlPullParser.START_TAG:
tagName = parser.getName();
if(tagName.equals("AdaptationSet")) {
currentMimeType = parser.getAttributeValue(XmlPullParser.NO_NAMESPACE, "mimeType");
} else if(tagName.equals("Representation") && currentMimeType.contains("audio")) {
currentBandwidth = Integer.parseInt(
parser.getAttributeValue(XmlPullParser.NO_NAMESPACE, "bandwidth"));
currentSamplingRate = Integer.parseInt(
parser.getAttributeValue(XmlPullParser.NO_NAMESPACE, "audioSamplingRate"));
} else if(tagName.equals("BaseURL")) {
currentTagIsBaseUrl = true;
}
break;
case XmlPullParser.TEXT:
if(currentTagIsBaseUrl &&
(currentMimeType.contains("audio"))) {
int format = -1;
if(currentMimeType.equals(MediaFormat.WEBMA.mimeType)) {
format = MediaFormat.WEBMA.id;
} else if(currentMimeType.equals(MediaFormat.M4A.mimeType)) {
format = MediaFormat.M4A.id;
}
audioStreams.add(new VideoInfo.AudioStream(parser.getText(),
format, currentBandwidth, currentSamplingRate));
}
break;
case XmlPullParser.END_TAG:
if(tagName.equals("AdaptationSet")) {
currentMimeType = "";
} else if(tagName.equals("BaseURL")) {
currentTagIsBaseUrl = false;
}//no break needed here
}
}
} catch(Exception e) {
throw new DashMpdParsingException("Could not parse Dash mpd", e);
}
return audioStreams;
}
}

41
app/src/main/java/org/schabi/newpipe/crawler/Downloader.java Normal file
View File

@ -0,0 +1,41 @@
package org.schabi.newpipe.crawler;
import java.io.IOException;
/**
* Created by Christian Schabesberger on 28.01.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* Downloader.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public interface Downloader {
/**Download the text file at the supplied URL as in download(String),
* but set the HTTP header field "Accept-Language" to the supplied string.
* @param siteUrl the URL of the text file to return the contents of
* @param language the language (usually a 2-character code) to set as the preferred language
* @return the contents of the specified text file
* @throws IOException*/
String download(String siteUrl, String language) throws IOException;
/**Download (via HTTP) the text file located at the supplied URL, and return its contents.
* Primarily intended for downloading web pages.
* @param siteUrl the URL of the text file to download
* @return the contents of the specified text file
* @throws IOException*/
String download(String siteUrl) throws IOException;
}

4
app/src/main/java/org/schabi/newpipe/services/MediaFormat.java → app/src/main/java/org/schabi/newpipe/crawler/MediaFormat.java
View File

@ -1,4 +1,4 @@
package org.schabi.newpipe.services;
package org.schabi.newpipe.crawler;
/**
* Created by Adam Howard on 08/11/15.
@ -6,7 +6,7 @@ package org.schabi.newpipe.services;
* Copyright (c) Christian Schabesberger <chris.schabesberger@mailbox.org>
* and Adam Howard <achdisposable1@gmail.com> 2015
*
* VideoListAdapter.java is part of NewPipe.
* MediaFormat.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by

35
app/src/main/java/org/schabi/newpipe/crawler/ParsingException.java Normal file
View File

@ -0,0 +1,35 @@
package org.schabi.newpipe.crawler;
/**
* Created by Christian Schabesberger on 31.01.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* ParsingException.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class ParsingException extends CrawlingException {
public ParsingException() {}
public ParsingException(String message) {
super(message);
}
public ParsingException(Throwable cause) {
super(cause);
}
public ParsingException(String message, Throwable cause) {
super(message, cause);
}
}

47
app/src/main/java/org/schabi/newpipe/crawler/RegexHelper.java Normal file
View File

@ -0,0 +1,47 @@
package org.schabi.newpipe.crawler;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Created by Christian Schabesberger on 02.02.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* RegexHelper.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
/** avoid using regex !!! */
public class RegexHelper {
public static class RegexException extends ParsingException {
public RegexException(String message) {
super(message);
}
}
public static String matchGroup1(String pattern, String input) throws RegexException {
Pattern pat = Pattern.compile(pattern);
Matcher mat = pat.matcher(input);
boolean foundMatch = mat.find();
if (foundMatch) {
return mat.group(1);
}
else {
//Log.e(TAG, "failed to find pattern \""+pattern+"\" inside of \""+input+"\"");
throw new RegexException("failed to find pattern \""+pattern+" inside of "+input+"\"");
}
}
}

16
app/src/main/java/org/schabi/newpipe/services/SearchEngine.java → app/src/main/java/org/schabi/newpipe/crawler/SearchEngine.java
View File

@ -1,8 +1,8 @@
package org.schabi.newpipe.services;
import org.schabi.newpipe.VideoPreviewInfo;
package org.schabi.newpipe.crawler;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Vector;
/**
@ -27,16 +27,16 @@ import java.util.Vector;
@SuppressWarnings("ALL")
public interface SearchEngine {
class Result {
public String errorMessage = "";
public String suggestion = "";
public final Vector<VideoPreviewInfo> resultList = new Vector<>();
public final List<VideoPreviewInfo> resultList = new Vector<>();
}
ArrayList<String> suggestionList(String query);
ArrayList<String> suggestionList(String query, Downloader dl)
throws CrawlingException, IOException;
//Result search(String query, int page);
Result search(String query, int page, String contentCountry);
Result search(String query, int page, String contentCountry, Downloader dl)
throws CrawlingException, IOException;
}

4
app/src/main/java/org/schabi/newpipe/services/ServiceList.java → app/src/main/java/org/schabi/newpipe/crawler/ServiceList.java
View File

@ -1,8 +1,8 @@
package org.schabi.newpipe.services;
package org.schabi.newpipe.crawler;
import android.util.Log;
import org.schabi.newpipe.services.youtube.YoutubeService;
import org.schabi.newpipe.crawler.services.youtube.YoutubeService;
/**
* Created by Christian Schabesberger on 23.08.15.

14
app/src/main/java/org/schabi/newpipe/services/StreamingService.java → app/src/main/java/org/schabi/newpipe/crawler/StreamingService.java
View File

@ -1,4 +1,6 @@
package org.schabi.newpipe.services;
package org.schabi.newpipe.crawler;
import java.io.IOException;
/**
* Created by Christian Schabesberger on 23.08.15.
@ -25,11 +27,11 @@ public interface StreamingService {
public String name = "";
}
ServiceInfo getServiceInfo();
VideoExtractor getExtractorInstance(String url);
VideoExtractor getExtractorInstance(String url, Downloader downloader)
throws IOException, CrawlingException;
SearchEngine getSearchEngineInstance();
/**When a VIEW_ACTION is caught this function will test if the url delivered within the calling
Intent was meant to be watched with this Service.
Return false if this service shall not allow to be called through ACTIONs.*/
boolean acceptUrl(String videoUrl);
VideoUrlIdHandler getUrlIdHandler();
}

77
app/src/main/java/org/schabi/newpipe/crawler/VideoExtractor.java Normal file
View File

@ -0,0 +1,77 @@
package org.schabi.newpipe.crawler;
/**
* Created by Christian Schabesberger on 10.08.15.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
* VideoExtractor.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
import java.util.List;
/**Scrapes information from a video streaming service (eg, YouTube).*/
@SuppressWarnings("ALL")
public interface VideoExtractor {
public class ExctractorInitException extends CrawlingException {
public ExctractorInitException() {}
public ExctractorInitException(String message) {
super(message);
}
public ExctractorInitException(Throwable cause) {
super(cause);
}
public ExctractorInitException(String message, Throwable cause) {
super(message, cause);
}
}
public class ContentNotAvailableException extends ParsingException {
public ContentNotAvailableException() {}
public ContentNotAvailableException(String message) {
super(message);
}
public ContentNotAvailableException(Throwable cause) {
super(cause);
}
public ContentNotAvailableException(String message, Throwable cause) {
super(message, cause);
}
}
public abstract int getTimeStamp() throws ParsingException;
public abstract String getTitle() throws ParsingException;
public abstract String getDescription() throws ParsingException;
public abstract String getUploader() throws ParsingException;
public abstract int getLength() throws ParsingException;
public abstract long getViews() throws ParsingException;
public abstract String getUploadDate() throws ParsingException;
public abstract String getThumbnailUrl() throws ParsingException;
public abstract String getUploaderThumbnailUrl() throws ParsingException;
public abstract List<VideoInfo.AudioStream> getAudioStreams() throws ParsingException;
public abstract List<VideoInfo.VideoStream> getVideoStreams() throws ParsingException;
public abstract String getDashMpdUrl() throws ParsingException;
public abstract int getAgeLimit() throws ParsingException;
public abstract String getAverageRating() throws ParsingException;
public abstract int getLikeCount() throws ParsingException;
public abstract int getDislikeCount() throws ParsingException;
public abstract VideoPreviewInfo getNextVideo() throws ParsingException;
public abstract List<VideoPreviewInfo> getRelatedVideos() throws ParsingException;
public abstract VideoUrlIdHandler getUrlIdConverter();
public abstract String getPageUrl();
}

79
app/src/main/java/org/schabi/newpipe/services/VideoInfo.java → app/src/main/java/org/schabi/newpipe/crawler/VideoInfo.java
View File

@ -1,9 +1,8 @@
package org.schabi.newpipe.services;
import org.schabi.newpipe.VideoPreviewInfo;
import org.schabi.newpipe.services.AbstractVideoInfo;
package org.schabi.newpipe.crawler;
import java.io.IOException;
import java.util.List;
import java.util.Vector;
/**
* Created by Christian Schabesberger on 26.08.15.
@ -29,20 +28,60 @@ import java.util.List;
@SuppressWarnings("ALL")
public class VideoInfo extends AbstractVideoInfo {
// If a video could not be parsed, this predefined error codes
// will be returned AND can be parsed by the frontend of the app.
// Error codes:
public final static int NO_ERROR = 0x0;
public final static int ERROR_NO_SPECIFIED_ERROR = 0x1;
// GEMA a german music colecting society.
public final static int ERROR_BLOCKED_BY_GEMA = 0x2;
/**Fills out the video info fields which are common to all services.
* Probably needs to be overridden by subclasses*/
public static VideoInfo getVideoInfo(VideoExtractor extractor, Downloader downloader)
throws CrawlingException, IOException {
VideoInfo videoInfo = new VideoInfo();
VideoUrlIdHandler uiconv = extractor.getUrlIdConverter();
videoInfo.webpage_url = extractor.getPageUrl();
videoInfo.title = extractor.getTitle();
videoInfo.duration = extractor.getLength();
videoInfo.uploader = extractor.getUploader();
videoInfo.description = extractor.getDescription();
videoInfo.view_count = extractor.getViews();
videoInfo.upload_date = extractor.getUploadDate();
videoInfo.thumbnail_url = extractor.getThumbnailUrl();
videoInfo.id = uiconv.getVideoId(extractor.getPageUrl());
videoInfo.dashMpdUrl = extractor.getDashMpdUrl();
/** Load and extract audio*/
videoInfo.audioStreams = extractor.getAudioStreams();
if(videoInfo.dashMpdUrl != null && !videoInfo.dashMpdUrl.isEmpty()) {
if(videoInfo.audioStreams == null) {
videoInfo.audioStreams = new Vector<AudioStream>();
}
videoInfo.audioStreams.addAll(
DashMpdParser.getAudioStreams(videoInfo.dashMpdUrl, downloader));
}
/** Extract video stream url*/
videoInfo.videoStreams = extractor.getVideoStreams();
videoInfo.uploader_thumbnail_url = extractor.getUploaderThumbnailUrl();
videoInfo.startPosition = extractor.getTimeStamp();
videoInfo.average_rating = extractor.getAverageRating();
videoInfo.like_count = extractor.getLikeCount();
videoInfo.dislike_count = extractor.getDislikeCount();
videoInfo.nextVideo = extractor.getNextVideo();
videoInfo.relatedVideos = extractor.getRelatedVideos();
//Bitmap thumbnail = null;
//Bitmap uploader_thumbnail = null;
//int videoAvailableStatus = VIDEO_AVAILABLE;
return videoInfo;
}
public String uploader_thumbnail_url = "";
public String description = "";
public VideoStream[] videoStreams = null;
public AudioStream[] audioStreams = null;
public int errorCode = NO_ERROR;
public String errorMessage = "";
/*todo: make this lists over vectors*/
public List<VideoStream> videoStreams = null;
public List<AudioStream> audioStreams = null;
// video streams provided by the dash mpd do not need to be provided as VideoStream.
// Later on this will also aplly to audio streams. Since dash mpd is standarized,
// crawling such a file is not service dependent. Therefore getting audio only streams by yust
// providing the dash mpd fille will be possible in the future.
public String dashMpdUrl = "";
public int duration = -1;
/*YouTube-specific fields
@ -53,11 +92,11 @@ public class VideoInfo extends AbstractVideoInfo {
public String average_rating = "";
public VideoPreviewInfo nextVideo = null;
public List<VideoPreviewInfo> relatedVideos = null;
public int startPosition = -1;//in seconds. some metadata is not passed using a VideoInfo object!
//in seconds. some metadata is not passed using a VideoInfo object!
public int startPosition = -1;
public VideoInfo() {}
/**Creates a new VideoInfo object from an existing AbstractVideoInfo.
* All the shared properties are copied to the new VideoInfo.*/
@SuppressWarnings("WeakerAccess")
@ -73,7 +112,8 @@ public class VideoInfo extends AbstractVideoInfo {
this.view_count = avi.view_count;
//todo: better than this
if(avi instanceof VideoPreviewInfo) {//shitty String to convert code
if(avi instanceof VideoPreviewInfo) {
//shitty String to convert code
String dur = ((VideoPreviewInfo)avi).duration;
int minutes = Integer.parseInt(dur.substring(0, dur.indexOf(":")));
int seconds = Integer.parseInt(dur.substring(dur.indexOf(":")+1, dur.length()));
@ -82,7 +122,8 @@ public class VideoInfo extends AbstractVideoInfo {
}
public static class VideoStream {
public String url = ""; //url of the stream
//url of the stream
public String url = "";
public int format = -1;
public String resolution = "";

4
app/src/main/java/org/schabi/newpipe/VideoPreviewInfo.java → app/src/main/java/org/schabi/newpipe/crawler/VideoPreviewInfo.java
View File

@ -1,10 +1,10 @@
package org.schabi.newpipe;
package org.schabi.newpipe.crawler;
import android.graphics.Bitmap;
import android.os.Parcel;
import android.os.Parcelable;
import org.schabi.newpipe.services.AbstractVideoInfo;
import org.schabi.newpipe.crawler.AbstractVideoInfo;
/**
* Created by Christian Schabesberger on 26.08.15.

32
app/src/main/java/org/schabi/newpipe/crawler/VideoUrlIdHandler.java Normal file
View File

@ -0,0 +1,32 @@
package org.schabi.newpipe.crawler;
/**
* Created by Christian Schabesberger on 02.02.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* VideoUrlIdHandler.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public interface VideoUrlIdHandler {
String getVideoUrl(String videoId);
String getVideoId(String siteUrl) throws ParsingException;
String cleanUrl(String siteUrl) throws ParsingException;
/**When a VIEW_ACTION is caught this function will test if the url delivered within the calling
Intent was meant to be watched with this Service.
Return false if this service shall not allow to be called through ACTIONs.*/
boolean acceptUrl(String videoUrl);
}

202
app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeSearchEngine.java Normal file
View File

@ -0,0 +1,202 @@
package org.schabi.newpipe.crawler.services.youtube;
import android.net.Uri;
import android.util.Log;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.crawler.CrawlingException;
import org.schabi.newpipe.crawler.Downloader;
import org.schabi.newpipe.crawler.ParsingException;
import org.schabi.newpipe.crawler.SearchEngine;
import org.schabi.newpipe.crawler.VideoExtractor;
import org.schabi.newpipe.crawler.VideoPreviewInfo;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
/**
* Created by Christian Schabesberger on 09.08.15.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
* YoutubeSearchEngine.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class YoutubeSearchEngine implements SearchEngine {
private static final String TAG = YoutubeSearchEngine.class.toString();
@Override
public Result search(String query, int page, String languageCode, Downloader downloader)
throws IOException, ParsingException {
Result result = new Result();
Uri.Builder builder = new Uri.Builder();
builder.scheme("https")
.authority("www.youtube.com")
.appendPath("results")
.appendQueryParameter("search_query", query)
.appendQueryParameter("page", Integer.toString(page))
.appendQueryParameter("filters", "video");
String site;
String url = builder.build().toString();
//if we've been passed a valid language code, append it to the URL
if(!languageCode.isEmpty()) {
//assert Pattern.matches("[a-z]{2}(-([A-Z]{2}|[0-9]{1,3}))?", languageCode);
site = downloader.download(url, languageCode);
}
else {
site = downloader.download(url);
}
try {
Document doc = Jsoup.parse(site, url);
Element list = doc.select("ol[class=\"item-section\"]").first();
for (Element item : list.children()) {
/* First we need to determine which kind of item we are working with.
Youtube depicts five different kinds of items on its search result page. These are
regular videos, playlists, channels, two types of video suggestions, and a "no video
found" item. Since we only want videos, we need to filter out all the others.
An example for this can be seen here:
https://www.youtube.com/results?search_query=asdf&page=1
We already applied a filter to the url, so we don't need to care about channels and
playlists now.
*/
Element el;
// both types of spell correction item
if (!((el = item.select("div[class*=\"spell-correction\"]").first()) == null)) {
result.suggestion = el.select("a").first().text();
// search message item
} else if (!((el = item.select("div[class*=\"search-message\"]").first()) == null)) {
result.errorMessage = el.text();
// video item type
} else if (!((el = item.select("div[class*=\"yt-lockup-video\"").first()) == null)) {
VideoPreviewInfo resultItem = new VideoPreviewInfo();
Element dl = el.select("h3").first().select("a").first();
resultItem.webpage_url = dl.attr("abs:href");
try {
Pattern p = Pattern.compile("v=([0-9a-zA-Z-]*)");
Matcher m = p.matcher(resultItem.webpage_url);
resultItem.id = m.group(1);
} catch (Exception e) {
//e.printStackTrace();
}
resultItem.title = dl.text();
resultItem.duration = item.select("span[class=\"video-time\"]").first().text();
resultItem.uploader = item.select("div[class=\"yt-lockup-byline\"]").first()
.select("a").first()
.text();
resultItem.upload_date = item.select("div[class=\"yt-lockup-meta\"]").first()
.select("li").first()
.text();
Element te = item.select("div[class=\"yt-thumb video-thumb\"]").first()
.select("img").first();
resultItem.thumbnail_url = te.attr("abs:src");
// Sometimes youtube sends links to gif files which somehow seem to not exist
// anymore. Items with such gif also offer a secondary image source. So we are going
// to use that if we've caught such an item.
if (resultItem.thumbnail_url.contains(".gif")) {
resultItem.thumbnail_url = te.attr("abs:data-thumb");
}
result.resultList.add(resultItem);
} else {
//noinspection ConstantConditions
Log.e(TAG, "unexpected element found:\"" + el + "\"");
}
}
} catch(Exception e) {
throw new ParsingException(e);
}
return result;
}
@Override
public ArrayList<String> suggestionList(String query, Downloader dl)
throws IOException, ParsingException {
ArrayList<String> suggestions = new ArrayList<>();
Uri.Builder builder = new Uri.Builder();
builder.scheme("https")
.authority("suggestqueries.google.com")
.appendPath("complete")
.appendPath("search")
.appendQueryParameter("client", "")
.appendQueryParameter("output", "toolbar")
.appendQueryParameter("ds", "yt")
.appendQueryParameter("q", query);
String url = builder.build().toString();
String response = dl.download(url);
try {
//TODO: Parse xml data using Jsoup not done
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
DocumentBuilder dBuilder;
org.w3c.dom.Document doc = null;
try {
dBuilder = dbFactory.newDocumentBuilder();
doc = dBuilder.parse(new InputSource(
new ByteArrayInputStream(response.getBytes("utf-8"))));
doc.getDocumentElement().normalize();
} catch (ParserConfigurationException | SAXException | IOException e) {
e.printStackTrace();
}
if (doc != null) {
NodeList nList = doc.getElementsByTagName("CompleteSuggestion");
for (int temp = 0; temp < nList.getLength(); temp++) {
NodeList nList1 = doc.getElementsByTagName("suggestion");
Node nNode1 = nList1.item(temp);
if (nNode1.getNodeType() == Node.ELEMENT_NODE) {
org.w3c.dom.Element eElement = (org.w3c.dom.Element) nNode1;
suggestions.add(eElement.getAttribute("data"));
}
}
} else {
Log.e(TAG, "GREAT FUCKING ERROR");
}
return suggestions;
} catch(Exception e) {
throw new ParsingException(e);
}
}
}

27
app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeService.java → app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeService.java
View File

@ -1,8 +1,13 @@
package org.schabi.newpipe.services.youtube;
package org.schabi.newpipe.crawler.services.youtube;
import org.schabi.newpipe.services.StreamingService;
import org.schabi.newpipe.services.VideoExtractor;
import org.schabi.newpipe.services.SearchEngine;
import org.schabi.newpipe.crawler.CrawlingException;
import org.schabi.newpipe.crawler.Downloader;
import org.schabi.newpipe.crawler.StreamingService;
import org.schabi.newpipe.crawler.VideoUrlIdHandler;
import org.schabi.newpipe.crawler.VideoExtractor;
import org.schabi.newpipe.crawler.SearchEngine;
import java.io.IOException;
/**
@ -33,9 +38,11 @@ public class YoutubeService implements StreamingService {
return serviceInfo;
}
@Override
public VideoExtractor getExtractorInstance(String url) {
if(acceptUrl(url)) {
return new YoutubeVideoExtractor(url);
public VideoExtractor getExtractorInstance(String url, Downloader downloader)
throws CrawlingException, IOException {
VideoUrlIdHandler urlIdHandler = new YoutubeVideoUrlIdHandler();
if(urlIdHandler.acceptUrl(url)) {
return new YoutubeVideoExtractor(url, downloader) ;
}
else {
throw new IllegalArgumentException("supplied String is not a valid Youtube URL");
@ -45,9 +52,9 @@ public class YoutubeService implements StreamingService {
public SearchEngine getSearchEngineInstance() {
return new YoutubeSearchEngine();
}
@Override
public boolean acceptUrl(String videoUrl) {
return videoUrl.contains("youtube") ||
videoUrl.contains("youtu.be");
public VideoUrlIdHandler getUrlIdHandler() {
return new YoutubeVideoUrlIdHandler();
}
}

600
app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java Normal file
View File

File diff suppressed because it is too large Load Diff

68
app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoUrlIdHandler.java Normal file
View File

@ -0,0 +1,68 @@
package org.schabi.newpipe.crawler.services.youtube;
import org.schabi.newpipe.crawler.ParsingException;
import org.schabi.newpipe.crawler.RegexHelper;
import org.schabi.newpipe.crawler.VideoUrlIdHandler;
/**
* Created by Christian Schabesberger on 02.02.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* YoutubeVideoUrlIdHandler.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class YoutubeVideoUrlIdHandler implements VideoUrlIdHandler {
@SuppressWarnings("WeakerAccess")
@Override
public String getVideoUrl(String videoId) {
return "https://www.youtube.com/watch?v=" + videoId;
}
@SuppressWarnings("WeakerAccess")
@Override
public String getVideoId(String url) throws ParsingException {
String id;
String pat;
if(url.contains("youtube")) {
pat = "youtube\\.com/watch\\?v=([\\-a-zA-Z0-9_]{11})";
}
else if(url.contains("youtu.be")) {
pat = "youtu\\.be/([a-zA-Z0-9_-]{11})";
}
else {
throw new ParsingException("Error no suitable url: " + url);
}
id = RegexHelper.matchGroup1(pat, url);
if(!id.isEmpty()){
//Log.i(TAG, "string \""+url+"\" matches!");
return id;
} else {
throw new ParsingException("Error could not parse url: " + url);
}
}
public String cleanUrl(String complexUrl) throws ParsingException {
return getVideoUrl(getVideoId(complexUrl));
}
@Override
public boolean acceptUrl(String videoUrl) {
return videoUrl.contains("youtube") ||
videoUrl.contains("youtu.be");
}
}

16
app/src/main/java/org/schabi/newpipe/services/AbstractVideoInfo.java
View File

@ -1,16 +0,0 @@
package org.schabi.newpipe.services;
import android.graphics.Bitmap;
/**Common properties between VideoInfo and VideoPreviewInfo.*/
public abstract class AbstractVideoInfo {
public String id = "";
public String title = "";
public String uploader = "";
//public int duration = -1;
public String thumbnail_url = "";
public Bitmap thumbnail = null;
public String webpage_url = "";
public String upload_date = "";
public long view_count = -1;
}

128
app/src/main/java/org/schabi/newpipe/services/VideoExtractor.java
View File

@ -1,128 +0,0 @@
package org.schabi.newpipe.services;
/**
* Created by Christian Schabesberger on 10.08.15.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
* VideoExtractor.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
/**Scrapes information from a video streaming service (eg, YouTube).*/
@SuppressWarnings("ALL")
public abstract class VideoExtractor {
protected final String pageUrl;
protected VideoInfo videoInfo;
@SuppressWarnings("WeakerAccess")
public VideoExtractor(String url) {
this.pageUrl = url;
}
/**Fills out the video info fields which are common to all services.
* Probably needs to be overridden by subclasses*/
public VideoInfo getVideoInfo()
{
if(videoInfo == null) {
videoInfo = new VideoInfo();
}
if(videoInfo.webpage_url.isEmpty()) {
videoInfo.webpage_url = pageUrl;
}
if(getErrorCode() == VideoInfo.NO_ERROR) {
if (videoInfo.title.isEmpty()) {
videoInfo.title = getTitle();
}
if (videoInfo.duration < 1) {
videoInfo.duration = getLength();
}
if (videoInfo.uploader.isEmpty()) {
videoInfo.uploader = getUploader();
}
if (videoInfo.description.isEmpty()) {
videoInfo.description = getDescription();
}
if (videoInfo.view_count == -1) {
videoInfo.view_count = getViews();
}
if (videoInfo.upload_date.isEmpty()) {
videoInfo.upload_date = getUploadDate();
}
if (videoInfo.thumbnail_url.isEmpty()) {
videoInfo.thumbnail_url = getThumbnailUrl();
}
if (videoInfo.id.isEmpty()) {
videoInfo.id = getVideoId(pageUrl);
}
/** Load and extract audio*/
if (videoInfo.audioStreams == null) {
videoInfo.audioStreams = getAudioStreams();
}
/** Extract video stream url*/
if (videoInfo.videoStreams == null) {
videoInfo.videoStreams = getVideoStreams();
}
if (videoInfo.uploader_thumbnail_url.isEmpty()) {
videoInfo.uploader_thumbnail_url = getUploaderThumbnailUrl();
}
if (videoInfo.startPosition < 0) {
videoInfo.startPosition = getTimeStamp();
}
} else {
videoInfo.errorCode = getErrorCode();
videoInfo.errorMessage = getErrorMessage();
}
//Bitmap thumbnail = null;
//Bitmap uploader_thumbnail = null;
//int videoAvailableStatus = VIDEO_AVAILABLE;
return videoInfo;
}
//todo: add licence field
public abstract int getErrorCode();
public abstract String getErrorMessage();
//todo: remove these functions, or make them static, otherwise its useles, to have them here
public abstract String getVideoUrl(String videoId);
public abstract String getVideoId(String siteUrl);
///////////////////////////////////////////////////////////////////////////////////////////
public abstract int getTimeStamp();
public abstract String getTitle();
public abstract String getDescription();
public abstract String getUploader();
public abstract int getLength();
public abstract long getViews();
public abstract String getUploadDate();
public abstract String getThumbnailUrl();
public abstract String getUploaderThumbnailUrl();
public abstract VideoInfo.AudioStream[] getAudioStreams();
public abstract VideoInfo.VideoStream[] getVideoStreams();
}

190
app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeSearchEngine.java
View File

@ -1,190 +0,0 @@
package org.schabi.newpipe.services.youtube;
import android.net.Uri;
import android.util.Log;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.services.SearchEngine;
import org.schabi.newpipe.VideoPreviewInfo;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
/**
* Created by Christian Schabesberger on 09.08.15.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
* YoutubeSearchEngine.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class YoutubeSearchEngine implements SearchEngine {
private static final String TAG = YoutubeSearchEngine.class.toString();
@Override
public Result search(String query, int page, String languageCode) {
//String contentCountry = PreferenceManager.getDefaultSharedPreferences(this).getString(getString(R.string., "");
Uri.Builder builder = new Uri.Builder();
builder.scheme("https")
.authority("www.youtube.com")
.appendPath("results")
.appendQueryParameter("search_query", query)
.appendQueryParameter("page", Integer.toString(page))
.appendQueryParameter("filters", "video");
String site;
String url = builder.build().toString();
//if we've been passed a valid language code, append it to the URL
if(!languageCode.isEmpty()) {
//assert Pattern.matches("[a-z]{2}(-([A-Z]{2}|[0-9]{1,3}))?", languageCode);
site = Downloader.download(url, languageCode);
}
else {
site = Downloader.download(url);
}
Document doc = Jsoup.parse(site, url);
Result result = new Result();
Element list = doc.select("ol[class=\"item-section\"]").first();
int i = 0;
for(Element item : list.children()) {
i++;
/* First we need to determine which kind of item we are working with.
Youtube depicts five different kinds of items on its search result page. These are
regular videos, playlists, channels, two types of video suggestions, and a "no video
found" item. Since we only want videos, we need to filter out all the others.
An example for this can be seen here:
https://www.youtube.com/results?search_query=asdf&page=1
We already applied a filter to the url, so we don't need to care about channels and
playlists now.
*/
Element el;
// both types of spell correction item
if(!((el = item.select("div[class*=\"spell-correction\"]").first()) == null)) {
result.suggestion = el.select("a").first().text();
// search message item
} else if(!((el = item.select("div[class*=\"search-message\"]").first()) == null)) {
result.errorMessage = el.text();
// video item type
} else if(!((el = item.select("div[class*=\"yt-lockup-video\"").first()) == null)) {
VideoPreviewInfo resultItem = new VideoPreviewInfo();
Element dl = el.select("h3").first().select("a").first();
resultItem.webpage_url = dl.attr("abs:href");
try {
Pattern p = Pattern.compile("v=([0-9a-zA-Z-]*)");
Matcher m = p.matcher(resultItem.webpage_url);
resultItem.id=m.group(1);
} catch (Exception e) {
//e.printStackTrace();
}
resultItem.title = dl.text();
resultItem.duration = item.select("span[class=\"video-time\"]").first().text();
resultItem.uploader = item.select("div[class=\"yt-lockup-byline\"]").first()
.select("a").first()
.text();
resultItem.upload_date = item.select("div[class=\"yt-lockup-meta\"]").first()
.select("li").first()
.text();
Element te = item.select("div[class=\"yt-thumb video-thumb\"]").first()
.select("img").first();
resultItem.thumbnail_url = te.attr("abs:src");
// Sometimes youtube sends links to gif files which somehow seem to not exist
// anymore. Items with such gif also offer a secondary image source. So we are going
// to use that if we've caught such an item.
if(resultItem.thumbnail_url.contains(".gif")) {
resultItem.thumbnail_url = te.attr("abs:data-thumb");
}
result.resultList.add(resultItem);
} else {
//noinspection ConstantConditions
Log.e(TAG, "unexpected element found:\""+el+"\"");
}
}
return result;
}
@Override
public ArrayList<String> suggestionList(String query) {
ArrayList<String> suggestions = new ArrayList<>();
Uri.Builder builder = new Uri.Builder();
builder.scheme("https")
.authority("suggestqueries.google.com")
.appendPath("complete")
.appendPath("search")
.appendQueryParameter("client", "")
.appendQueryParameter("output", "toolbar")
.appendQueryParameter("ds", "yt")
.appendQueryParameter("q", query);
String url = builder.build().toString();
String response = Downloader.download(url);
//TODO: Parse xml data using Jsoup not done
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
DocumentBuilder dBuilder;
org.w3c.dom.Document doc = null;
try {
dBuilder = dbFactory.newDocumentBuilder();
doc = dBuilder.parse(new InputSource(new ByteArrayInputStream(response.getBytes("utf-8"))));
doc.getDocumentElement().normalize();
}catch (ParserConfigurationException | SAXException | IOException e) {
e.printStackTrace();
}
if(doc!=null){
NodeList nList = doc.getElementsByTagName("CompleteSuggestion");
for (int temp = 0; temp < nList.getLength(); temp++) {
NodeList nList1 = doc.getElementsByTagName("suggestion");
Node nNode1 = nList1.item(temp);
if (nNode1.getNodeType() == Node.ELEMENT_NODE) {
org.w3c.dom.Element eElement = (org.w3c.dom.Element) nNode1;
suggestions.add(eElement.getAttribute("data"));
}
}
}else {
Log.e(TAG, "GREAT FUCKING ERROR");
}
return suggestions;
}
}

647
app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractor.java
View File

File diff suppressed because it is too large Load Diff

6
app/src/main/res/values/strings.xml
View File

@ -65,7 +65,13 @@
<string name="background_player_playing_toast">Playing in background</string>
<string name="c3s_url" translatable="false">https://www.c3s.cc/</string>
<string name="play_btn_text">Play</string>
<string name="general_error">Error</string>
<string name="network_error">Network error</string>
<string name="could_not_load_thumbnails">Could not load Thumbnails</string>
<string name="youtube_signature_decryption_error">Could not decrypt video url signature.</string>
<string name="parsing_error">Could not parse website.</string>
<string name="content_not_available">Content not available.</string>
<string name="blocked_by_gema">Blocked by GEMA.</string>
<!-- Content descriptions (for better accessibility) -->
<string name="list_thumbnail_view_description">Video preview thumbnail</string>