diff --git a/lib/linguist/blob_helper.rb b/lib/linguist/blob_helper.rb
index 05755730..8fd46759 100644
--- a/lib/linguist/blob_helper.rb
+++ b/lib/linguist/blob_helper.rb
@@ -92,8 +92,22 @@ module Linguist
     #
     # Return true or false
     def binary?
-      return false if data.empty?
-      encoding.nil? || detect_encoding[:type] == :binary
+      # Large blobs aren't even loaded into memory
+      if data.nil?
+        true
+
+      # Treat blank files as text
+      elsif data == ""
+        false
+
+      # Charlock doesn't know what to think
+      elsif encoding.nil?
+        true
+
+      # If Charlock says its binary
+      else
+        detect_encoding[:type] == :binary
+      end
     end
 
     # Public: Is the blob text?
diff --git a/test/test_blob.rb b/test/test_blob.rb
index 5a741099..712dfa19 100644
--- a/test/test_blob.rb
+++ b/test/test_blob.rb
@@ -84,6 +84,13 @@ class TestBlob < Test::Unit::TestCase
   end
 
   def test_binary
+    # Large blobs aren't loaded
+    large_blob = blob("git.exe")
+    large_blob.instance_eval do
+      def data; end
+    end
+    assert large_blob.binary?
+
     assert blob("git.deb").binary?
     assert blob("git.exe").binary?
     assert blob("hello.pbc").binary?