Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
tianlh
LightGBM-DCU
Commits
28481763
Commit
28481763
authored
Feb 27, 2018
by
Qiwei Ye
Committed by
Guolin Ke
Feb 27, 2018
Browse files
minor change for #1243 (#1250)
* minor change * unix line break
parent
5dcb4b90
Changes
1
Show whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
12 additions
and
12 deletions
+12
-12
src/io/file_io.cpp
src/io/file_io.cpp
+12
-12
No files found.
src/io/file_io.cpp
View file @
28481763
...
@@ -51,9 +51,9 @@ private:
...
@@ -51,9 +51,9 @@ private:
const
std
::
string
kHdfsProto
=
"hdfs://"
;
const
std
::
string
kHdfsProto
=
"hdfs://"
;
#ifdef USE_HDFS
#ifdef USE_HDFS
struct
H
dfs
File
:
VirtualFileReader
,
VirtualFileWriter
{
struct
H
DFS
File
:
VirtualFileReader
,
VirtualFileWriter
{
H
dfs
File
(
const
std
::
string
&
filename
,
int
flags
)
:
filename_
(
filename
),
flags_
(
flags
)
{}
H
DFS
File
(
const
std
::
string
&
filename
,
int
flags
)
:
filename_
(
filename
),
flags_
(
flags
)
{}
~
H
dfs
File
()
{
~
H
DFS
File
()
{
if
(
file_
!=
NULL
)
{
if
(
file_
!=
NULL
)
{
hdfsCloseFile
(
fs_
,
file_
);
hdfsCloseFile
(
fs_
,
file_
);
}
}
...
@@ -62,7 +62,7 @@ struct HdfsFile : VirtualFileReader, VirtualFileWriter {
...
@@ -62,7 +62,7 @@ struct HdfsFile : VirtualFileReader, VirtualFileWriter {
bool
Init
()
{
bool
Init
()
{
if
(
file_
==
NULL
)
{
if
(
file_
==
NULL
)
{
if
(
fs_
==
NULL
)
{
if
(
fs_
==
NULL
)
{
fs_
=
g
etH
dfsFS
(
filename_
);
fs_
=
G
etH
DFSFileSystem
(
filename_
);
}
}
if
(
fs_
!=
NULL
&&
(
flags_
==
O_WRONLY
||
0
==
hdfsExists
(
fs_
,
filename_
.
c_str
())))
{
if
(
fs_
!=
NULL
&&
(
flags_
==
O_WRONLY
||
0
==
hdfsExists
(
fs_
,
filename_
.
c_str
())))
{
file_
=
hdfsOpenFile
(
fs_
,
filename_
.
c_str
(),
flags_
,
0
,
0
,
0
);
file_
=
hdfsOpenFile
(
fs_
,
filename_
.
c_str
(),
flags_
,
0
,
0
,
0
);
...
@@ -73,7 +73,7 @@ struct HdfsFile : VirtualFileReader, VirtualFileWriter {
...
@@ -73,7 +73,7 @@ struct HdfsFile : VirtualFileReader, VirtualFileWriter {
bool
Exists
()
const
{
bool
Exists
()
const
{
if
(
fs_
==
NULL
)
{
if
(
fs_
==
NULL
)
{
fs_
=
g
etH
dfsFS
(
filename_
);
fs_
=
G
etH
DFSFileSystem
(
filename_
);
}
}
return
fs_
!=
NULL
&&
0
==
hdfsExists
(
fs_
,
filename_
.
c_str
());
return
fs_
!=
NULL
&&
0
==
hdfsExists
(
fs_
,
filename_
.
c_str
());
}
}
...
@@ -110,7 +110,7 @@ private:
...
@@ -110,7 +110,7 @@ private:
return
bytes
-
remain
;
return
bytes
-
remain
;
}
}
static
hdfsFS
g
etH
dfsFS
(
const
std
::
string
&
uri
)
{
static
hdfsFS
G
etH
DFSFileSystem
(
const
std
::
string
&
uri
)
{
size_t
end
=
uri
.
find
(
"/"
,
kHdfsProto
.
length
());
size_t
end
=
uri
.
find
(
"/"
,
kHdfsProto
.
length
());
if
(
uri
.
find
(
kHdfsProto
)
!=
0
||
end
==
std
::
string
::
npos
)
{
if
(
uri
.
find
(
kHdfsProto
)
!=
0
||
end
==
std
::
string
::
npos
)
{
Log
::
Warning
(
"Bad hdfs uri, no namenode found [%s]"
,
uri
.
c_str
());
Log
::
Warning
(
"Bad hdfs uri, no namenode found [%s]"
,
uri
.
c_str
());
...
@@ -118,12 +118,12 @@ private:
...
@@ -118,12 +118,12 @@ private:
}
}
std
::
string
hostport
=
uri
.
substr
(
kHdfsProto
.
length
(),
end
-
kHdfsProto
.
length
());
std
::
string
hostport
=
uri
.
substr
(
kHdfsProto
.
length
(),
end
-
kHdfsProto
.
length
());
if
(
fs_cache_
.
count
(
hostport
)
==
0
)
{
if
(
fs_cache_
.
count
(
hostport
)
==
0
)
{
fs_cache_
[
hostport
]
=
m
akeH
dfsFs
(
hostport
);
fs_cache_
[
hostport
]
=
M
akeH
DFSFileSystem
(
hostport
);
}
}
return
fs_cache_
[
hostport
];
return
fs_cache_
[
hostport
];
}
}
static
hdfsFS
m
akeH
dfsFs
(
const
std
::
string
&
hostport
)
{
static
hdfsFS
M
akeH
DFSFileSystem
(
const
std
::
string
&
hostport
)
{
std
::
istringstream
iss
(
hostport
);
std
::
istringstream
iss
(
hostport
);
std
::
string
host
;
std
::
string
host
;
tPort
port
=
0
;
tPort
port
=
0
;
...
@@ -143,7 +143,7 @@ private:
...
@@ -143,7 +143,7 @@ private:
static
std
::
unordered_map
<
std
::
string
,
hdfsFS
>
fs_cache_
;
static
std
::
unordered_map
<
std
::
string
,
hdfsFS
>
fs_cache_
;
};
};
std
::
unordered_map
<
std
::
string
,
hdfsFS
>
H
dfs
File
::
fs_cache_
=
std
::
unordered_map
<
std
::
string
,
hdfsFS
>
();
std
::
unordered_map
<
std
::
string
,
hdfsFS
>
H
DFS
File
::
fs_cache_
=
std
::
unordered_map
<
std
::
string
,
hdfsFS
>
();
#define WITH_HDFS(x) x
#define WITH_HDFS(x) x
#else
#else
...
@@ -152,7 +152,7 @@ std::unordered_map<std::string, hdfsFS> HdfsFile::fs_cache_ = std::unordered_map
...
@@ -152,7 +152,7 @@ std::unordered_map<std::string, hdfsFS> HdfsFile::fs_cache_ = std::unordered_map
std
::
unique_ptr
<
VirtualFileReader
>
VirtualFileReader
::
Make
(
const
std
::
string
&
filename
)
{
std
::
unique_ptr
<
VirtualFileReader
>
VirtualFileReader
::
Make
(
const
std
::
string
&
filename
)
{
if
(
0
==
filename
.
find
(
kHdfsProto
))
{
if
(
0
==
filename
.
find
(
kHdfsProto
))
{
WITH_HDFS
(
return
std
::
unique_ptr
<
VirtualFileReader
>
(
new
H
dfs
File
(
filename
,
O_RDONLY
)));
WITH_HDFS
(
return
std
::
unique_ptr
<
VirtualFileReader
>
(
new
H
DFS
File
(
filename
,
O_RDONLY
)));
}
else
{
}
else
{
return
std
::
unique_ptr
<
VirtualFileReader
>
(
new
LocalFile
(
filename
,
"rb"
));
return
std
::
unique_ptr
<
VirtualFileReader
>
(
new
LocalFile
(
filename
,
"rb"
));
}
}
...
@@ -160,7 +160,7 @@ std::unique_ptr<VirtualFileReader> VirtualFileReader::Make(const std::string& fi
...
@@ -160,7 +160,7 @@ std::unique_ptr<VirtualFileReader> VirtualFileReader::Make(const std::string& fi
std
::
unique_ptr
<
VirtualFileWriter
>
VirtualFileWriter
::
Make
(
const
std
::
string
&
filename
)
{
std
::
unique_ptr
<
VirtualFileWriter
>
VirtualFileWriter
::
Make
(
const
std
::
string
&
filename
)
{
if
(
0
==
filename
.
find
(
kHdfsProto
))
{
if
(
0
==
filename
.
find
(
kHdfsProto
))
{
WITH_HDFS
(
return
std
::
unique_ptr
<
VirtualFileWriter
>
(
new
H
dfs
File
(
filename
,
O_WRONLY
)));
WITH_HDFS
(
return
std
::
unique_ptr
<
VirtualFileWriter
>
(
new
H
DFS
File
(
filename
,
O_WRONLY
)));
}
else
{
}
else
{
return
std
::
unique_ptr
<
VirtualFileWriter
>
(
new
LocalFile
(
filename
,
"wb"
));
return
std
::
unique_ptr
<
VirtualFileWriter
>
(
new
LocalFile
(
filename
,
"wb"
));
}
}
...
@@ -168,7 +168,7 @@ std::unique_ptr<VirtualFileWriter> VirtualFileWriter::Make(const std::string& fi
...
@@ -168,7 +168,7 @@ std::unique_ptr<VirtualFileWriter> VirtualFileWriter::Make(const std::string& fi
bool
VirtualFileWriter
::
Exists
(
const
std
::
string
&
filename
)
{
bool
VirtualFileWriter
::
Exists
(
const
std
::
string
&
filename
)
{
if
(
0
==
filename
.
find
(
kHdfsProto
))
{
if
(
0
==
filename
.
find
(
kHdfsProto
))
{
WITH_HDFS
(
H
dfs
File
file
(
filename
,
O_RDONLY
);
return
file
.
Exists
());
WITH_HDFS
(
H
DFS
File
file
(
filename
,
O_RDONLY
);
return
file
.
Exists
());
}
else
{
}
else
{
LocalFile
file
(
filename
,
"rb"
);
LocalFile
file
(
filename
,
"rb"
);
return
file
.
Exists
();
return
file
.
Exists
();
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment