diff --git a/README.md b/README.md
index 15ba3d04b..0d615f43c 100644
--- a/README.md
+++ b/README.md
@@ -12,7 +12,7 @@
-Screenshots • Description • Features • Contribution • Donate • License
+Screenshots • Description • Features • Updates • Contribution • Donate • License
Website • Blog • Press
@@ -73,6 +73,20 @@ NewPipe does not use any Google framework libraries, nor the YouTube API. Websit
* Show comments
* … and many more
+## Updates
+When a change to the NewPipe code occurs (due to either adding features or bug fixing), eventually a release will occur. These are in the format x.xx.x . In order to get this new version, you can:
+ * Build a debug APK yourself. This is the fastest way to get new features on your device, but is much more complicated, so we recommend using one of the other methods.
+ * Download the APK from [releases](https://github.com/TeamNewPipe/NewPipe/releases) and install it.
+ * Update via F-droid. This is the slowest method of getting updates, as F-Droid must recognize changes, build the APK itself, sign it, then push the update to users.
+
+When you install an APK from one of these options, it will be incompatible with an APK from one of the other options. This is due to different signing keys being used. Signing keys help ensure that a user isn't tricked into installing a malicious update to an app, and are independent. F-Droid and GitHub use different signing keys, and building an APK debug excludes a key. The signing key issue is being discussed in issue [#1981](https://github.com/TeamNewPipe/NewPipe/issues/1981), and may be fixed by setting up our own repository on F-Droid.
+
+In the meanwhile, if you want to switch sources for some reason (e.g. NewPipe's core functionality was broken and F-Droid doesn't have the update yet), we recommend following this procedure:
+1. Back up your data via "Settings>Content>Export Database" so you keep your history, subscriptions, and playlists
+2. Uninstall NewPipe
+3. Download the APK from the new source and install it
+4. Import the data from step 1 via "Settings>Content>Import Database"
+
## Contribution
Whether you have ideas, translations, design changes, code cleaning, or real heavy code changes, help is always welcome.
The more is done the better it gets!
diff --git a/app/build.gradle b/app/build.gradle
index bb993cca0..782634a0b 100644
--- a/app/build.gradle
+++ b/app/build.gradle
@@ -8,18 +8,20 @@ android {
applicationId "org.schabi.newpipe"
minSdkVersion 19
targetSdkVersion 28
- versionCode 70
- versionName "0.15.0"
+ versionCode 71
+ versionName "0.15.1"
testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner"
vectorDrawables.useSupportLibrary = true
}
+
buildTypes {
release {
minifyEnabled true
shrinkResources true
proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro'
}
+
debug {
multiDexEnabled true
debuggable true
@@ -33,6 +35,7 @@ android {
// but continue the build even when errors are found:
abortOnError false
}
+
compileOptions {
sourceCompatibility JavaVersion.VERSION_1_8
targetCompatibility JavaVersion.VERSION_1_8
@@ -54,7 +57,7 @@ dependencies {
exclude module: 'support-annotations'
})
- implementation 'com.github.TeamNewPipe:NewPipeExtractor:99915e4527c0'
+ implementation 'com.github.TeamNewPipe:NewPipeExtractor:f7c7b9df1a'
testImplementation 'junit:junit:4.12'
testImplementation 'org.mockito:mockito-core:2.23.0'
diff --git a/app/src/androidTest/java/org/schabi/newpipe/ApplicationTest.java b/app/src/androidTest/java/org/schabi/newpipe/ApplicationTest.java
deleted file mode 100644
index d5b0b7087..000000000
--- a/app/src/androidTest/java/org/schabi/newpipe/ApplicationTest.java
+++ /dev/null
@@ -1,13 +0,0 @@
-package org.schabi.newpipe;
-
-import android.app.Application;
-import android.test.ApplicationTestCase;
-
-/**
- * Testing Fundamentals
- */
-public class ApplicationTest extends ApplicationTestCase {
- public ApplicationTest() {
- super(Application.class);
- }
-}
\ No newline at end of file
diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml
index 1bc205f33..5aa1dc982 100644
--- a/app/src/main/AndroidManifest.xml
+++ b/app/src/main/AndroidManifest.xml
@@ -119,7 +119,6 @@
-
[]
@@ -88,6 +90,8 @@ public class App extends Application {
}
refWatcher = installLeakCanary();
+ app = this;
+
// Initialize settings first because others inits can use its values
SettingsActivity.initSettings(this);
@@ -100,6 +104,9 @@ public class App extends Application {
ImageLoader.getInstance().init(getImageLoaderConfigurations(10, 50));
configureRxJavaErrorHandler();
+
+ // Check for new version
+ new CheckForNewAppVersionTask().execute();
}
protected Downloader getDownloader() {
@@ -211,6 +218,31 @@ public class App extends Application {
NotificationManager mNotificationManager =
(NotificationManager) getSystemService(Context.NOTIFICATION_SERVICE);
mNotificationManager.createNotificationChannel(mChannel);
+
+ setUpUpdateNotificationChannel(importance);
+ }
+
+ /**
+ * Set up notification channel for app update.
+ * @param importance
+ */
+ @TargetApi(Build.VERSION_CODES.O)
+ private void setUpUpdateNotificationChannel(int importance) {
+
+ final String appUpdateId
+ = getString(R.string.app_update_notification_channel_id);
+ final CharSequence appUpdateName
+ = getString(R.string.app_update_notification_channel_name);
+ final String appUpdateDescription
+ = getString(R.string.app_update_notification_channel_description);
+
+ NotificationChannel appUpdateChannel
+ = new NotificationChannel(appUpdateId, appUpdateName, importance);
+ appUpdateChannel.setDescription(appUpdateDescription);
+
+ NotificationManager appUpdateNotificationManager
+ = (NotificationManager) getSystemService(Context.NOTIFICATION_SERVICE);
+ appUpdateNotificationManager.createNotificationChannel(appUpdateChannel);
}
@Nullable
@@ -226,4 +258,8 @@ public class App extends Application {
protected boolean isDisposedRxExceptionsReported() {
return false;
}
+
+ public static App getApp() {
+ return app;
+ }
}
diff --git a/app/src/main/java/org/schabi/newpipe/CheckForNewAppVersionTask.java b/app/src/main/java/org/schabi/newpipe/CheckForNewAppVersionTask.java
new file mode 100644
index 000000000..af9b88ac1
--- /dev/null
+++ b/app/src/main/java/org/schabi/newpipe/CheckForNewAppVersionTask.java
@@ -0,0 +1,230 @@
+package org.schabi.newpipe;
+
+import android.app.Application;
+import android.app.PendingIntent;
+import android.content.Intent;
+import android.content.SharedPreferences;
+import android.content.pm.PackageInfo;
+import android.content.pm.PackageManager;
+import android.content.pm.Signature;
+import android.net.Uri;
+import android.os.AsyncTask;
+import android.preference.PreferenceManager;
+import android.support.v4.app.NotificationCompat;
+import android.support.v4.app.NotificationManagerCompat;
+
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.schabi.newpipe.report.ErrorActivity;
+import org.schabi.newpipe.report.UserAction;
+
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.security.cert.CertificateEncodingException;
+import java.security.cert.CertificateException;
+import java.security.cert.CertificateFactory;
+import java.security.cert.X509Certificate;
+import java.util.concurrent.TimeUnit;
+
+import okhttp3.OkHttpClient;
+import okhttp3.Request;
+import okhttp3.RequestBody;
+import okhttp3.Response;
+
+/**
+ * AsyncTask to check if there is a newer version of the NewPipe github apk available or not.
+ * If there is a newer version we show a notification, informing the user. On tapping
+ * the notification, the user will be directed to the download link.
+ */
+public class CheckForNewAppVersionTask extends AsyncTask {
+
+ private static final Application app = App.getApp();
+ private static final String GITHUB_APK_SHA1 = "B0:2E:90:7C:1C:D6:FC:57:C3:35:F0:88:D0:8F:50:5F:94:E4:D2:15";
+ private static final String newPipeApiUrl = "https://newpipe.schabi.org/api/data.json";
+ private static final int timeoutPeriod = 30;
+
+ private SharedPreferences mPrefs;
+ private OkHttpClient client;
+
+ @Override
+ protected void onPreExecute() {
+
+ mPrefs = PreferenceManager.getDefaultSharedPreferences(app);
+
+ // Check if user has enabled/ disabled update checking
+ // and if the current apk is a github one or not.
+ if (!mPrefs.getBoolean(app.getString(R.string.update_app_key), true)
+ || !isGithubApk()) {
+ this.cancel(true);
+ }
+ }
+
+ @Override
+ protected String doInBackground(Void... voids) {
+
+ // Make a network request to get latest NewPipe data.
+ if (client == null) {
+
+ client = new OkHttpClient
+ .Builder()
+ .readTimeout(timeoutPeriod, TimeUnit.SECONDS)
+ .build();
+ }
+
+ Request request = new Request.Builder()
+ .url(newPipeApiUrl)
+ .build();
+
+ try {
+ Response response = client.newCall(request).execute();
+ return response.body().string();
+ } catch (IOException ex) {
+ ErrorActivity.reportError(app, ex, null, null,
+ ErrorActivity.ErrorInfo.make(UserAction.SOMETHING_ELSE, "none",
+ "app update API fail", R.string.app_ui_crash));
+ }
+
+ return null;
+ }
+
+ @Override
+ protected void onPostExecute(String response) {
+
+ // Parse the json from the response.
+ if (response != null) {
+
+ try {
+ JSONObject mainObject = new JSONObject(response);
+ JSONObject flavoursObject = mainObject.getJSONObject("flavors");
+ JSONObject githubObject = flavoursObject.getJSONObject("github");
+ JSONObject githubStableObject = githubObject.getJSONObject("stable");
+
+ String versionName = githubStableObject.getString("version");
+ String versionCode = githubStableObject.getString("version_code");
+ String apkLocationUrl = githubStableObject.getString("apk");
+
+ compareAppVersionAndShowNotification(versionName, apkLocationUrl, versionCode);
+
+ } catch (JSONException ex) {
+ ErrorActivity.reportError(app, ex, null, null,
+ ErrorActivity.ErrorInfo.make(UserAction.SOMETHING_ELSE, "none",
+ "could not parse app update JSON data", R.string.app_ui_crash));
+ }
+ }
+ }
+
+ /**
+ * Method to compare the current and latest available app version.
+ * If a newer version is available, we show the update notification.
+ * @param versionName
+ * @param apkLocationUrl
+ */
+ private void compareAppVersionAndShowNotification(String versionName,
+ String apkLocationUrl,
+ String versionCode) {
+
+ int NOTIFICATION_ID = 2000;
+
+ if (BuildConfig.VERSION_CODE < Integer.valueOf(versionCode)) {
+
+ // A pending intent to open the apk location url in the browser.
+ Intent intent = new Intent(Intent.ACTION_VIEW, Uri.parse(apkLocationUrl));
+ PendingIntent pendingIntent
+ = PendingIntent.getActivity(app, 0, intent, 0);
+
+ NotificationCompat.Builder notificationBuilder = new NotificationCompat
+ .Builder(app, app.getString(R.string.app_update_notification_channel_id))
+ .setSmallIcon(R.drawable.ic_newpipe_update)
+ .setVisibility(NotificationCompat.VISIBILITY_PUBLIC)
+ .setContentIntent(pendingIntent)
+ .setAutoCancel(true)
+ .setContentTitle(app.getString(R.string.app_update_notification_content_title))
+ .setContentText(app.getString(R.string.app_update_notification_content_text)
+ + " " + versionName);
+
+ NotificationManagerCompat notificationManager = NotificationManagerCompat.from(app);
+ notificationManager.notify(NOTIFICATION_ID, notificationBuilder.build());
+ }
+ }
+
+ /**
+ * Method to get the apk's SHA1 key.
+ * https://stackoverflow.com/questions/9293019/get-certificate-fingerprint-from-android-app#22506133
+ */
+ private static String getCertificateSHA1Fingerprint() {
+
+ PackageManager pm = app.getPackageManager();
+ String packageName = app.getPackageName();
+ int flags = PackageManager.GET_SIGNATURES;
+ PackageInfo packageInfo = null;
+
+ try {
+ packageInfo = pm.getPackageInfo(packageName, flags);
+ } catch (PackageManager.NameNotFoundException ex) {
+ ErrorActivity.reportError(app, ex, null, null,
+ ErrorActivity.ErrorInfo.make(UserAction.SOMETHING_ELSE, "none",
+ "Could not find package info", R.string.app_ui_crash));
+ }
+
+ Signature[] signatures = packageInfo.signatures;
+ byte[] cert = signatures[0].toByteArray();
+ InputStream input = new ByteArrayInputStream(cert);
+
+ CertificateFactory cf = null;
+ X509Certificate c = null;
+
+ try {
+ cf = CertificateFactory.getInstance("X509");
+ c = (X509Certificate) cf.generateCertificate(input);
+ } catch (CertificateException ex) {
+ ErrorActivity.reportError(app, ex, null, null,
+ ErrorActivity.ErrorInfo.make(UserAction.SOMETHING_ELSE, "none",
+ "Certificate error", R.string.app_ui_crash));
+ }
+
+ String hexString = null;
+
+ try {
+ MessageDigest md = MessageDigest.getInstance("SHA1");
+ byte[] publicKey = md.digest(c.getEncoded());
+ hexString = byte2HexFormatted(publicKey);
+ } catch (NoSuchAlgorithmException ex1) {
+ ErrorActivity.reportError(app, ex1, null, null,
+ ErrorActivity.ErrorInfo.make(UserAction.SOMETHING_ELSE, "none",
+ "Could not retrieve SHA1 key", R.string.app_ui_crash));
+ } catch (CertificateEncodingException ex2) {
+ ErrorActivity.reportError(app, ex2, null, null,
+ ErrorActivity.ErrorInfo.make(UserAction.SOMETHING_ELSE, "none",
+ "Could not retrieve SHA1 key", R.string.app_ui_crash));
+ }
+
+ return hexString;
+ }
+
+ private static String byte2HexFormatted(byte[] arr) {
+
+ StringBuilder str = new StringBuilder(arr.length * 2);
+
+ for (int i = 0; i < arr.length; i++) {
+ String h = Integer.toHexString(arr[i]);
+ int l = h.length();
+ if (l == 1) h = "0" + h;
+ if (l > 2) h = h.substring(l - 2, l);
+ str.append(h.toUpperCase());
+ if (i < (arr.length - 1)) str.append(':');
+ }
+ return str.toString();
+ }
+
+ public static boolean isGithubApk() {
+
+ return getCertificateSHA1Fingerprint().equals(GITHUB_APK_SHA1);
+ }
+}
diff --git a/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java b/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java
index 4f98f7f28..ec6d42b29 100644
--- a/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java
+++ b/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java
@@ -457,7 +457,7 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck
break;
case R.id.subtitle_button:
stream = subtitleStreamsAdapter.getItem(selectedSubtitleIndex);
- location = NewPipeSettings.getVideoDownloadPath(context);// assume that subtitle & video go together
+ location = NewPipeSettings.getVideoDownloadPath(context);// assume that subtitle & video files go together
kind = 's';
break;
default:
@@ -477,7 +477,6 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck
final String finalFileName = fileName;
DownloadManagerService.checkForRunningMission(context, location, fileName, (listed, finished) -> {
- // should be safe run the following code without "getActivity().runOnUiThread()"
if (listed) {
AlertDialog.Builder builder = new AlertDialog.Builder(context);
builder.setTitle(R.string.download_dialog_title)
@@ -511,11 +510,11 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck
if (secondaryStream != null) {
secondaryStreamUrl = secondaryStream.getStream().getUrl();
- psName = selectedStream.getFormat() == MediaFormat.MPEG_4 ? Postprocessing.ALGORITHM_MP4_DASH_MUXER : Postprocessing.ALGORITHM_WEBM_MUXER;
+ psName = selectedStream.getFormat() == MediaFormat.MPEG_4 ? Postprocessing.ALGORITHM_MP4_MUXER : Postprocessing.ALGORITHM_WEBM_MUXER;
psArgs = null;
long videoSize = wrappedVideoStreams.getSizeInBytes((VideoStream) selectedStream);
- // set nearLength, only, if both sizes are fetched or known. this probably does not work on weak internet connections
+ // set nearLength, only, if both sizes are fetched or known. this probably does not work on slow networks
if (secondaryStream.getSizeInBytes() > 0 && videoSize > 0) {
nearLength = secondaryStream.getSizeInBytes() + videoSize;
}
diff --git a/app/src/main/java/org/schabi/newpipe/download/ExtSDDownloadFailedActivity.java b/app/src/main/java/org/schabi/newpipe/download/ExtSDDownloadFailedActivity.java
deleted file mode 100644
index c02ef92eb..000000000
--- a/app/src/main/java/org/schabi/newpipe/download/ExtSDDownloadFailedActivity.java
+++ /dev/null
@@ -1,38 +0,0 @@
-package org.schabi.newpipe.download;
-
-import android.app.AlertDialog;
-import android.content.DialogInterface;
-import android.os.Bundle;
-import android.support.annotation.Nullable;
-import android.support.v7.app.AppCompatActivity;
-
-import org.schabi.newpipe.R;
-import org.schabi.newpipe.settings.NewPipeSettings;
-import org.schabi.newpipe.util.ServiceHelper;
-import org.schabi.newpipe.util.ThemeHelper;
-
-public class ExtSDDownloadFailedActivity extends AppCompatActivity {
- @Override
- protected void onCreate(@Nullable Bundle savedInstanceState) {
- super.onCreate(savedInstanceState);
- ThemeHelper.setTheme(this, ServiceHelper.getSelectedServiceId(this));
- }
-
- @Override
- protected void onStart() {
- super.onStart();
- new AlertDialog.Builder(this)
- .setTitle(R.string.download_to_sdcard_error_title)
- .setMessage(R.string.download_to_sdcard_error_message)
- .setPositiveButton(R.string.yes, (DialogInterface dialogInterface, int i) -> {
- NewPipeSettings.resetDownloadFolders(this);
- finish();
- })
- .setNegativeButton(R.string.cancel, (DialogInterface dialogInterface, int i) -> {
- dialogInterface.dismiss();
- finish();
- })
- .create()
- .show();
- }
-}
diff --git a/app/src/main/java/org/schabi/newpipe/settings/MainSettingsFragment.java b/app/src/main/java/org/schabi/newpipe/settings/MainSettingsFragment.java
index 5e07e2b12..43270926e 100644
--- a/app/src/main/java/org/schabi/newpipe/settings/MainSettingsFragment.java
+++ b/app/src/main/java/org/schabi/newpipe/settings/MainSettingsFragment.java
@@ -4,6 +4,7 @@ import android.os.Bundle;
import android.support.v7.preference.Preference;
import org.schabi.newpipe.BuildConfig;
+import org.schabi.newpipe.CheckForNewAppVersionTask;
import org.schabi.newpipe.R;
public class MainSettingsFragment extends BasePreferenceFragment {
@@ -13,6 +14,13 @@ public class MainSettingsFragment extends BasePreferenceFragment {
public void onCreatePreferences(Bundle savedInstanceState, String rootKey) {
addPreferencesFromResource(R.xml.main_settings);
+ if (!CheckForNewAppVersionTask.isGithubApk()) {
+ final Preference update = findPreference(getString(R.string.update_pref_screen_key));
+ getPreferenceScreen().removePreference(update);
+
+ defaultPreferences.edit().putBoolean(getString(R.string.update_app_key), false).apply();
+ }
+
if (!DEBUG) {
final Preference debug = findPreference(getString(R.string.debug_pref_screen_key));
getPreferenceScreen().removePreference(debug);
diff --git a/app/src/main/java/org/schabi/newpipe/settings/UpdateSettingsFragment.java b/app/src/main/java/org/schabi/newpipe/settings/UpdateSettingsFragment.java
new file mode 100644
index 000000000..76d887dd1
--- /dev/null
+++ b/app/src/main/java/org/schabi/newpipe/settings/UpdateSettingsFragment.java
@@ -0,0 +1,33 @@
+package org.schabi.newpipe.settings;
+
+import android.os.Bundle;
+import android.support.annotation.Nullable;
+import android.support.v7.preference.Preference;
+
+import org.schabi.newpipe.CheckForNewAppVersionTask;
+import org.schabi.newpipe.R;
+
+public class UpdateSettingsFragment extends BasePreferenceFragment {
+
+ @Override
+ public void onCreate(@Nullable Bundle savedInstanceState) {
+ super.onCreate(savedInstanceState);
+
+ String updateToggleKey = getString(R.string.update_app_key);
+ findPreference(updateToggleKey).setOnPreferenceChangeListener(updatePreferenceChange);
+ }
+
+ @Override
+ public void onCreatePreferences(Bundle savedInstanceState, String rootKey) {
+ addPreferencesFromResource(R.xml.update_settings);
+ }
+
+ private Preference.OnPreferenceChangeListener updatePreferenceChange
+ = (preference, newValue) -> {
+
+ defaultPreferences.edit().putBoolean(getString(R.string.update_app_key),
+ (boolean) newValue).apply();
+
+ return true;
+ };
+}
diff --git a/app/src/main/java/org/schabi/newpipe/util/SecondaryStreamHelper.java b/app/src/main/java/org/schabi/newpipe/util/SecondaryStreamHelper.java
index a5d3ea3eb..b3522aea0 100644
--- a/app/src/main/java/org/schabi/newpipe/util/SecondaryStreamHelper.java
+++ b/app/src/main/java/org/schabi/newpipe/util/SecondaryStreamHelper.java
@@ -36,7 +36,6 @@ public class SecondaryStreamHelper {
* @return selected audio stream or null if a candidate was not found
*/
public static AudioStream getAudioStreamFor(@NonNull List audioStreams, @NonNull VideoStream videoStream) {
- // TODO: check if m4v and m4a selected streams are DASH compliant
switch (videoStream.getFormat()) {
case WEBM:
case MPEG_4:
diff --git a/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java b/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java
index ce7ae267c..b864cf4fb 100644
--- a/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java
+++ b/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java
@@ -156,7 +156,6 @@ public class DownloadInitializer extends Thread {
if (retryCount++ > mMission.maxRetry) {
Log.e(TAG, "initializer failed", e);
- mMission.running = false;
mMission.notifyError(e);
return;
}
diff --git a/app/src/main/java/us/shandian/giga/get/DownloadMission.java b/app/src/main/java/us/shandian/giga/get/DownloadMission.java
index c25d517f1..243a8585a 100644
--- a/app/src/main/java/us/shandian/giga/get/DownloadMission.java
+++ b/app/src/main/java/us/shandian/giga/get/DownloadMission.java
@@ -39,7 +39,7 @@ public class DownloadMission extends Mission {
public static final int ERROR_SSL_EXCEPTION = 1004;
public static final int ERROR_UNKNOWN_HOST = 1005;
public static final int ERROR_CONNECT_HOST = 1006;
- public static final int ERROR_POSTPROCESSING_FAILED = 1007;
+ public static final int ERROR_POSTPROCESSING = 1007;
public static final int ERROR_HTTP_NO_CONTENT = 204;
public static final int ERROR_HTTP_UNSUPPORTED_RANGE = 206;
@@ -79,9 +79,12 @@ public class DownloadMission extends Mission {
public String postprocessingName;
/**
- * Indicates if the post-processing algorithm is actually running, used to detect corrupt downloads
+ * Indicates if the post-processing state:
+ * 0: ready
+ * 1: running
+ * 2: completed
*/
- public boolean postprocessingRunning;
+ public int postprocessingState;
/**
* Indicate if the post-processing algorithm works on the same file
@@ -356,7 +359,7 @@ public class DownloadMission extends Mission {
finishCount++;
if (finishCount == currentThreadCount) {
- if (errCode > ERROR_NOTHING) return;
+ if (errCode != ERROR_NOTHING) return;
if (DEBUG) {
Log.d(TAG, "onFinish" + (current + 1) + "/" + urls.length);
@@ -382,19 +385,26 @@ public class DownloadMission extends Mission {
}
}
- private void notifyPostProcessing(boolean processing) {
+ private void notifyPostProcessing(int state) {
if (DEBUG) {
- Log.d(TAG, (processing ? "enter" : "exit") + " postprocessing on " + location + File.separator + name);
+ String action;
+ switch (state) {
+ case 1:
+ action = "Running";
+ break;
+ case 2:
+ action = "Completed";
+ break;
+ default:
+ action = "Failed";
+ }
+
+ Log.d(TAG, action + " postprocessing on " + location + File.separator + name);
}
synchronized (blockState) {
- if (!processing) {
- postprocessingName = null;
- postprocessingArgs = null;
- }
-
// don't return without fully write the current state
- postprocessingRunning = processing;
+ postprocessingState = state;
Utility.writeToFile(metadata, DownloadMission.this);
}
}
@@ -403,16 +413,30 @@ public class DownloadMission extends Mission {
* Start downloading with multiple threads.
*/
public void start() {
- if (running || current >= urls.length) return;
+ if (running || isFinished()) return;
// ensure that the previous state is completely paused.
joinForThread(init);
- for (Thread thread : threads) joinForThread(thread);
+ if (threads != null)
+ for (Thread thread : threads) joinForThread(thread);
enqueued = false;
running = true;
errCode = ERROR_NOTHING;
+ if (current >= urls.length && postprocessingName != null) {
+ runAsync(1, () -> {
+ if (doPostprocessing()) {
+ running = false;
+ deleteThisFromFile();
+
+ notify(DownloadManagerService.MESSAGE_FINISHED);
+ }
+ });
+
+ return;
+ }
+
if (blocks < 0) {
initializer();
return;
@@ -420,7 +444,7 @@ public class DownloadMission extends Mission {
init = null;
- if (threads.length < 1) {
+ if (threads == null || threads.length < 1) {
threads = new Thread[currentThreadCount];
}
@@ -444,18 +468,18 @@ public class DownloadMission extends Mission {
public synchronized void pause() {
if (!running) return;
- running = false;
- recovered = true;
- enqueued = false;
-
- if (postprocessingRunning) {
+ if (isPsRunning()) {
if (DEBUG) {
Log.w(TAG, "pause during post-processing is not applicable.");
}
return;
}
- if (init != null && init.isAlive()) {
+ running = false;
+ recovered = true;
+ enqueued = false;
+
+ if (init != null && Thread.currentThread() != init && init.isAlive()) {
init.interrupt();
synchronized (blockState) {
resetState();
@@ -532,13 +556,36 @@ public class DownloadMission extends Mission {
mWritingToFile = false;
}
+ /**
+ * Indicates if the download if fully finished
+ *
+ * @return true, otherwise, false
+ */
public boolean isFinished() {
- return current >= urls.length && postprocessingName == null;
+ return current >= urls.length && (postprocessingName == null || postprocessingState == 2);
+ }
+
+ /**
+ * Indicates if the download file is corrupt due a failed post-processing
+ *
+ * @return {@code true} if this mission is unrecoverable
+ */
+ public boolean isPsFailed() {
+ return postprocessingName != null && errCode == DownloadMission.ERROR_POSTPROCESSING && postprocessingThis;
+ }
+
+ /**
+ * Indicates if a post-processing algorithm is running
+ *
+ * @return true, otherwise, false
+ */
+ public boolean isPsRunning() {
+ return postprocessingName != null && postprocessingState == 1;
}
public long getLength() {
long calculated;
- if (postprocessingRunning) {
+ if (postprocessingState == 1) {
calculated = length;
} else {
calculated = offsets[current < offsets.length ? current : (offsets.length - 1)] + length;
@@ -550,16 +597,19 @@ public class DownloadMission extends Mission {
}
private boolean doPostprocessing() {
- if (postprocessingName == null) return true;
+ if (postprocessingName == null || postprocessingState == 2) return true;
+
+ notifyPostProcessing(1);
+ notifyProgress(0);
+
+ Thread.currentThread().setName("[" + TAG + "] post-processing = " + postprocessingName + " filename = " + name);
+
+ Exception exception = null;
try {
- notifyPostProcessing(true);
- notifyProgress(0);
-
- Thread.currentThread().setName("[" + TAG + "] post-processing = " + postprocessingName + " filename = " + name);
-
- Postprocessing algorithm = Postprocessing.getAlgorithm(postprocessingName, this);
- algorithm.run();
+ Postprocessing
+ .getAlgorithm(postprocessingName, this)
+ .run();
} catch (Exception err) {
StringBuilder args = new StringBuilder(" ");
if (postprocessingArgs != null) {
@@ -571,15 +621,21 @@ public class DownloadMission extends Mission {
}
Log.e(TAG, String.format("Post-processing failed. algorithm = %s args = [%s]", postprocessingName, args), err);
- notifyError(ERROR_POSTPROCESSING_FAILED, err);
- return false;
+ if (errCode == ERROR_NOTHING) errCode = ERROR_POSTPROCESSING;
+
+ exception = err;
} finally {
- notifyPostProcessing(false);
+ notifyPostProcessing(errCode == ERROR_NOTHING ? 2 : 0);
}
- if (errCode != ERROR_NOTHING) notify(DownloadManagerService.MESSAGE_ERROR);
+ if (errCode != ERROR_NOTHING) {
+ if (exception == null) exception = errObject;
+ notifyError(ERROR_POSTPROCESSING, exception);
- return errCode == ERROR_NOTHING;
+ return false;
+ }
+
+ return true;
}
private boolean deleteThisFromFile() {
diff --git a/app/src/main/java/us/shandian/giga/postprocessing/Mp4DashMuxer.java b/app/src/main/java/us/shandian/giga/postprocessing/Mp4DashMuxer.java
index b303b66cd..45c06dd4b 100644
--- a/app/src/main/java/us/shandian/giga/postprocessing/Mp4DashMuxer.java
+++ b/app/src/main/java/us/shandian/giga/postprocessing/Mp4DashMuxer.java
@@ -13,9 +13,7 @@ import us.shandian.giga.get.DownloadMission;
class Mp4DashMuxer extends Postprocessing {
Mp4DashMuxer(DownloadMission mission) {
- super(mission);
- recommendedReserve = 15360 * 1024;// 15 MiB
- worksOnSameFile = true;
+ super(mission, 15360 * 1024/* 15 MiB */, true);
}
@Override
diff --git a/app/src/main/java/us/shandian/giga/postprocessing/Mp4Muxer.java b/app/src/main/java/us/shandian/giga/postprocessing/Mp4Muxer.java
new file mode 100644
index 000000000..bf932d5c1
--- /dev/null
+++ b/app/src/main/java/us/shandian/giga/postprocessing/Mp4Muxer.java
@@ -0,0 +1,136 @@
+package us.shandian.giga.postprocessing;
+
+import android.media.MediaCodec.BufferInfo;
+import android.media.MediaExtractor;
+import android.media.MediaMuxer;
+import android.media.MediaMuxer.OutputFormat;
+import android.util.Log;
+
+import static org.schabi.newpipe.BuildConfig.DEBUG;
+
+import org.schabi.newpipe.streams.io.SharpStream;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import us.shandian.giga.get.DownloadMission;
+
+
+class Mp4Muxer extends Postprocessing {
+ private static final String TAG = "Mp4Muxer";
+ private static final int NOTIFY_BYTES_INTERVAL = 128 * 1024;// 128 KiB
+
+ Mp4Muxer(DownloadMission mission) {
+ super(mission, 0, false);
+ }
+
+ @Override
+ int process(SharpStream out, SharpStream... sources) throws IOException {
+ File dlFile = mission.getDownloadedFile();
+ File tmpFile = new File(mission.location, mission.name.concat(".tmp"));
+
+ if (tmpFile.exists())
+ if (!tmpFile.delete()) return DownloadMission.ERROR_FILE_CREATION;
+
+ if (!tmpFile.createNewFile()) return DownloadMission.ERROR_FILE_CREATION;
+
+ FileInputStream source = null;
+ MediaMuxer muxer = null;
+
+ //noinspection TryFinallyCanBeTryWithResources
+ try {
+ source = new FileInputStream(dlFile);
+ MediaExtractor tracks[] = {
+ getMediaExtractor(source, mission.offsets[0], mission.offsets[1] - mission.offsets[0]),
+ getMediaExtractor(source, mission.offsets[1], mission.length - mission.offsets[1])
+ };
+
+ muxer = new MediaMuxer(tmpFile.getAbsolutePath(), OutputFormat.MUXER_OUTPUT_MPEG_4);
+
+ int tracksIndex[] = {
+ muxer.addTrack(tracks[0].getTrackFormat(0)),
+ muxer.addTrack(tracks[1].getTrackFormat(0))
+ };
+
+ ByteBuffer buffer = ByteBuffer.allocate(512 * 1024);// 512 KiB
+ BufferInfo info = new BufferInfo();
+
+ long written = 0;
+ long nextReport = NOTIFY_BYTES_INTERVAL;
+
+ muxer.start();
+
+ while (true) {
+ int done = 0;
+
+ for (int i = 0; i < tracks.length; i++) {
+ if (tracksIndex[i] < 0) continue;
+
+ info.set(0,
+ tracks[i].readSampleData(buffer, 0),
+ tracks[i].getSampleTime(),
+ tracks[i].getSampleFlags()
+ );
+
+ if (info.size >= 0) {
+ muxer.writeSampleData(tracksIndex[i], buffer, info);
+ written += info.size;
+ done++;
+ }
+ if (!tracks[i].advance()) {
+ // EOF reached
+ tracks[i].release();
+ tracksIndex[i] = -1;
+ }
+
+ if (written > nextReport) {
+ nextReport = written + NOTIFY_BYTES_INTERVAL;
+ super.progressReport(written);
+ }
+ }
+
+ if (done < 1) break;
+ }
+
+ // this part should not fail
+ if (!dlFile.delete()) return DownloadMission.ERROR_FILE_CREATION;
+ if (!tmpFile.renameTo(dlFile)) return DownloadMission.ERROR_FILE_CREATION;
+
+ return OK_RESULT;
+ } finally {
+ try {
+ if (muxer != null) {
+ muxer.stop();
+ muxer.release();
+ }
+ } catch (Exception err) {
+ if (DEBUG)
+ Log.e(TAG, "muxer stop/release failed", err);
+ }
+
+ if (source != null) {
+ try {
+ source.close();
+ } catch (IOException e) {
+ // nothing to do
+ }
+ }
+
+ // if the operation fails, delete the temporal file
+ if (tmpFile.exists()) {
+ //noinspection ResultOfMethodCallIgnored
+ tmpFile.delete();
+ }
+ }
+ }
+
+ private MediaExtractor getMediaExtractor(FileInputStream source, long offset, long length) throws IOException {
+ MediaExtractor extractor = new MediaExtractor();
+ extractor.setDataSource(source.getFD(), offset, length);
+ extractor.selectTrack(0);
+
+ return extractor;
+ }
+}
diff --git a/app/src/main/java/us/shandian/giga/postprocessing/Postprocessing.java b/app/src/main/java/us/shandian/giga/postprocessing/Postprocessing.java
index 80726f705..635140bd3 100644
--- a/app/src/main/java/us/shandian/giga/postprocessing/Postprocessing.java
+++ b/app/src/main/java/us/shandian/giga/postprocessing/Postprocessing.java
@@ -18,21 +18,21 @@ public abstract class Postprocessing {
public static final String ALGORITHM_TTML_CONVERTER = "ttml";
public static final String ALGORITHM_MP4_DASH_MUXER = "mp4D";
+ public static final String ALGORITHM_MP4_MUXER = "mp4";
public static final String ALGORITHM_WEBM_MUXER = "webm";
- private static final String ALGORITHM_TEST_ALGO = "test";
public static Postprocessing getAlgorithm(String algorithmName, DownloadMission mission) {
if (null == algorithmName) {
throw new NullPointerException("algorithmName");
} else switch (algorithmName) {
case ALGORITHM_TTML_CONVERTER:
- return new TttmlConverter(mission);
+ return new TtmlConverter(mission);
case ALGORITHM_MP4_DASH_MUXER:
return new Mp4DashMuxer(mission);
+ case ALGORITHM_MP4_MUXER:
+ return new Mp4Muxer(mission);
case ALGORITHM_WEBM_MUXER:
return new WebMMuxer(mission);
- case ALGORITHM_TEST_ALGO:
- return new TestAlgo(mission);
/*case "example-algorithm":
return new ExampleAlgorithm(mission);*/
default:
@@ -52,71 +52,84 @@ public abstract class Postprocessing {
*/
public int recommendedReserve;
+ /**
+ * the download to post-process
+ */
protected DownloadMission mission;
- Postprocessing(DownloadMission mission) {
+ Postprocessing(DownloadMission mission, int recommendedReserve, boolean worksOnSameFile) {
this.mission = mission;
+ this.recommendedReserve = recommendedReserve;
+ this.worksOnSameFile = worksOnSameFile;
}
public void run() throws IOException {
File file = mission.getDownloadedFile();
CircularFile out = null;
- ChunkFileInputStream[] sources = new ChunkFileInputStream[mission.urls.length];
+ int result;
+ long finalLength = -1;
- try {
- int i = 0;
- for (; i < sources.length - 1; i++) {
- sources[i] = new ChunkFileInputStream(file, mission.offsets[i], mission.offsets[i + 1], "rw");
- }
- sources[i] = new ChunkFileInputStream(file, mission.offsets[i], mission.getDownloadedFile().length(), "rw");
+ mission.done = 0;
+ mission.length = file.length();
- int[] idx = {0};
- CircularFile.OffsetChecker checker = () -> {
- while (idx[0] < sources.length) {
- /*
- * WARNING: never use rewind() in any chunk after any writing (especially on first chunks)
- * or the CircularFile can lead to unexpected results
- */
- if (sources[idx[0]].isDisposed() || sources[idx[0]].available() < 1) {
- idx[0]++;
- continue;// the selected source is not used anymore
+ if (worksOnSameFile) {
+ ChunkFileInputStream[] sources = new ChunkFileInputStream[mission.urls.length];
+ try {
+ int i = 0;
+ for (; i < sources.length - 1; i++) {
+ sources[i] = new ChunkFileInputStream(file, mission.offsets[i], mission.offsets[i + 1], "rw");
+ }
+ sources[i] = new ChunkFileInputStream(file, mission.offsets[i], mission.getDownloadedFile().length(), "rw");
+
+ int[] idx = {0};
+ CircularFile.OffsetChecker checker = () -> {
+ while (idx[0] < sources.length) {
+ /*
+ * WARNING: never use rewind() in any chunk after any writing (especially on first chunks)
+ * or the CircularFile can lead to unexpected results
+ */
+ if (sources[idx[0]].isDisposed() || sources[idx[0]].available() < 1) {
+ idx[0]++;
+ continue;// the selected source is not used anymore
+ }
+
+ return sources[idx[0]].getFilePointer() - 1;
}
- return sources[idx[0]].getFilePointer() - 1;
+ return -1;
+ };
+ out = new CircularFile(file, 0, this::progressReport, checker);
+
+ result = process(out, sources);
+
+ if (result == OK_RESULT)
+ finalLength = out.finalizeFile();
+ } finally {
+ for (SharpStream source : sources) {
+ if (source != null && !source.isDisposed()) {
+ source.dispose();
+ }
}
-
- return -1;
- };
-
- out = new CircularFile(file, 0, this::progressReport, checker);
-
- mission.done = 0;
- mission.length = file.length();
-
- int result = process(out, sources);
-
- if (result == OK_RESULT) {
- long finalLength = out.finalizeFile();
- mission.done = finalLength;
- mission.length = finalLength;
- } else {
- mission.errCode = DownloadMission.ERROR_UNKNOWN_EXCEPTION;
- mission.errObject = new RuntimeException("post-processing algorithm returned " + result);
- }
-
- if (result != OK_RESULT && worksOnSameFile) {
- //noinspection ResultOfMethodCallIgnored
- new File(mission.location, mission.name).delete();
- }
- } finally {
- for (SharpStream source : sources) {
- if (source != null && !source.isDisposed()) {
- source.dispose();
+ if (out != null) {
+ out.dispose();
}
}
- if (out != null) {
- out.dispose();
- }
+ } else {
+ result = process(null);
+ }
+
+ if (result == OK_RESULT) {
+ if (finalLength < 0) finalLength = file.length();
+ mission.done = finalLength;
+ mission.length = finalLength;
+ } else {
+ mission.errCode = DownloadMission.ERROR_UNKNOWN_EXCEPTION;
+ mission.errObject = new RuntimeException("post-processing algorithm returned " + result);
+ }
+
+ if (result != OK_RESULT && worksOnSameFile) {
+ //noinspection ResultOfMethodCallIgnored
+ file.delete();
}
}
@@ -138,7 +151,7 @@ public abstract class Postprocessing {
return mission.postprocessingArgs[index];
}
- private void progressReport(long done) {
+ void progressReport(long done) {
mission.done = done;
if (mission.length < mission.done) mission.length = mission.done;
diff --git a/app/src/main/java/us/shandian/giga/postprocessing/TestAlgo.java b/app/src/main/java/us/shandian/giga/postprocessing/TestAlgo.java
deleted file mode 100644
index 66b235d7c..000000000
--- a/app/src/main/java/us/shandian/giga/postprocessing/TestAlgo.java
+++ /dev/null
@@ -1,54 +0,0 @@
-package us.shandian.giga.postprocessing;
-
-import android.util.Log;
-
-import org.schabi.newpipe.streams.io.SharpStream;
-
-import java.io.IOException;
-import java.util.Random;
-
-import us.shandian.giga.get.DownloadMission;
-
-/**
- * Algorithm for testing proposes
- */
-class TestAlgo extends Postprocessing {
-
- public TestAlgo(DownloadMission mission) {
- super(mission);
-
- worksOnSameFile = true;
- recommendedReserve = 4096 * 1024;// 4 KiB
- }
-
- @Override
- int process(SharpStream out, SharpStream... sources) throws IOException {
-
- int written = 0;
- int size = 5 * 1024 * 1024;// 5 MiB
- byte[] buffer = new byte[8 * 1024];//8 KiB
- mission.length = size;
-
- Random rnd = new Random();
-
- // only write random data
- sources[0].dispose();
-
- while (written < size) {
- rnd.nextBytes(buffer);
-
- int read = Math.min(buffer.length, size - written);
- out.write(buffer, 0, read);
-
- try {
- Thread.sleep((int) (Math.random() * 10));
- } catch (InterruptedException e) {
- return -1;
- }
-
- written += read;
- }
-
- return Postprocessing.OK_RESULT;
- }
-}
diff --git a/app/src/main/java/us/shandian/giga/postprocessing/TttmlConverter.java b/app/src/main/java/us/shandian/giga/postprocessing/TtmlConverter.java
similarity index 83%
rename from app/src/main/java/us/shandian/giga/postprocessing/TttmlConverter.java
rename to app/src/main/java/us/shandian/giga/postprocessing/TtmlConverter.java
index 4c9d44548..390061840 100644
--- a/app/src/main/java/us/shandian/giga/postprocessing/TttmlConverter.java
+++ b/app/src/main/java/us/shandian/giga/postprocessing/TtmlConverter.java
@@ -18,13 +18,12 @@ import us.shandian.giga.postprocessing.io.SharpInputStream;
/**
* @author kapodamy
*/
-class TttmlConverter extends Postprocessing {
- private static final String TAG = "TttmlConverter";
+class TtmlConverter extends Postprocessing {
+ private static final String TAG = "TtmlConverter";
- TttmlConverter(DownloadMission mission) {
- super(mission);
- recommendedReserve = 0;// due how XmlPullParser works, the xml is fully loaded on the ram
- worksOnSameFile = true;
+ TtmlConverter(DownloadMission mission) {
+ // due how XmlPullParser works, the xml is fully loaded on the ram
+ super(mission, 0, true);
}
@Override
@@ -41,7 +40,7 @@ class TttmlConverter extends Postprocessing {
out,
getArgumentAt(1, "true").equals("true"),
getArgumentAt(2, "true").equals("true")
- );
+ );
} catch (Exception err) {
Log.e(TAG, "subtitle parse failed", err);
@@ -56,7 +55,7 @@ class TttmlConverter extends Postprocessing {
} else if (err instanceof XPathExpressionException) {
return 7;
}
-
+
return 8;
}
diff --git a/app/src/main/java/us/shandian/giga/postprocessing/WebMMuxer.java b/app/src/main/java/us/shandian/giga/postprocessing/WebMMuxer.java
index 009a9a66b..2ffb0f08d 100644
--- a/app/src/main/java/us/shandian/giga/postprocessing/WebMMuxer.java
+++ b/app/src/main/java/us/shandian/giga/postprocessing/WebMMuxer.java
@@ -15,9 +15,7 @@ import us.shandian.giga.get.DownloadMission;
class WebMMuxer extends Postprocessing {
WebMMuxer(DownloadMission mission) {
- super(mission);
- recommendedReserve = 2048 * 1024;// 2 MiB
- worksOnSameFile = true;
+ super(mission, 2048 * 1024/* 2 MiB */, true);
}
@Override
diff --git a/app/src/main/java/us/shandian/giga/service/DownloadManager.java b/app/src/main/java/us/shandian/giga/service/DownloadManager.java
index 6bcf84745..883c26850 100644
--- a/app/src/main/java/us/shandian/giga/service/DownloadManager.java
+++ b/app/src/main/java/us/shandian/giga/service/DownloadManager.java
@@ -141,15 +141,18 @@ public class DownloadManager {
File dl = mis.getDownloadedFile();
boolean exists = dl.exists();
- if (mis.postprocessingRunning && mis.postprocessingThis) {
- // Incomplete post-processing results in a corrupted download file
- // because the selected algorithm works on the same file to save space.
- if (!dl.delete()) {
- Log.w(TAG, "Unable to delete incomplete download file: " + sub.getPath());
+ if (mis.isPsRunning()) {
+ if (mis.postprocessingThis) {
+ // Incomplete post-processing results in a corrupted download file
+ // because the selected algorithm works on the same file to save space.
+ if (exists && dl.isFile() && !dl.delete())
+ Log.w(TAG, "Unable to delete incomplete download file: " + sub.getPath());
+
+ exists = true;
}
- exists = true;
- mis.postprocessingRunning = false;
- mis.errCode = DownloadMission.ERROR_POSTPROCESSING_FAILED;
+
+ mis.postprocessingState = 0;
+ mis.errCode = DownloadMission.ERROR_POSTPROCESSING;
mis.errObject = new RuntimeException("stopped unexpectedly");
} else if (exists && !dl.isFile()) {
// probably a folder, this should never happens
@@ -332,7 +335,7 @@ public class DownloadManager {
int count = 0;
synchronized (this) {
for (DownloadMission mission : mMissionsPending) {
- if (mission.running && mission.errCode != DownloadMission.ERROR_POSTPROCESSING_FAILED && !mission.isFinished())
+ if (mission.running && !mission.isFinished() && !mission.isPsFailed())
count++;
}
}
@@ -471,7 +474,7 @@ public class DownloadManager {
boolean flag = false;
synchronized (this) {
for (DownloadMission mission : mMissionsPending) {
- if (mission.running && mission.isFinished() && !mission.postprocessingRunning) {
+ if (mission.running && !mission.isFinished() && !mission.isPsRunning()) {
flag = true;
mission.pause();
}
@@ -528,6 +531,8 @@ public class DownloadManager {
ArrayList